diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2021-05-20 09:47:09 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2021-06-07 11:15:42 +0000 |
commit | 189d4fd8fad9e3c776873be51938cd31a42b6177 (patch) | |
tree | 6497caeff5e383937996768766ab3bb2081a40b2 /chromium/components/reporting | |
parent | 8bc75099d364490b22f43a7ce366b366c08f4164 (diff) | |
download | qtwebengine-chromium-189d4fd8fad9e3c776873be51938cd31a42b6177.tar.gz |
BASELINE: Update Chromium to 90.0.4430.221
Change-Id: Iff4d9d18d2fcf1a576f3b1f453010f744a232920
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/components/reporting')
68 files changed, 12420 insertions, 0 deletions
diff --git a/chromium/components/reporting/OWNERS b/chromium/components/reporting/OWNERS new file mode 100644 index 00000000000..9160064e786 --- /dev/null +++ b/chromium/components/reporting/OWNERS @@ -0,0 +1,3 @@ +lbaraz@chromium.com +zatrudo@google.com + diff --git a/chromium/components/reporting/README.md b/chromium/components/reporting/README.md new file mode 100644 index 00000000000..8c8e9dd738f --- /dev/null +++ b/chromium/components/reporting/README.md @@ -0,0 +1,16 @@ +The Encrypted Reporting Pipeline (ERP) provides a universal method for upload of +data for enterprise customers. + +The code structure looks like this: +Chrome: + - //components/reporting + Code shared between Chrome and Chrome OS. + - //chrome/browser/policy/messaging_layer + Code that lives only in the browser, primary interfaces for reporting data + such as ReportQueue and ReportQueueConfiguration. +Chrome OS: + - //platform2/missived + Daemon for encryption and storage of reports. + +If you'd like to begin using ERP within Chrome please check the comment in +[//chrome/browser/policy/messaging_layer/public/report_client.h](https:://chromium.googlesource.com/chromium/src/+/master/chrome/browser/policy/messaging_layer/public/report_client.h#25). diff --git a/chromium/components/reporting/encryption/BUILD.gn b/chromium/components/reporting/encryption/BUILD.gn new file mode 100644 index 00000000000..c1b13212ba6 --- /dev/null +++ b/chromium/components/reporting/encryption/BUILD.gn @@ -0,0 +1,108 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD - style license that can be +# found in the LICENSE file. + +import("//build/config/features.gni") + +static_library("encryption_module") { + sources = [ + "encryption_module.cc", + "encryption_module.h", + ] + deps = [ + ":encryption", + "//base", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + ] +} + +static_library("encryption") { + sources = [ + "encryption.cc", + "encryption.h", + ] + deps = [ + "//base", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + "//crypto", + "//crypto:platform", + "//third_party/boringssl", + ] +} + +static_library("decryption") { + sources = [ + "decryption.cc", + "decryption.h", + ] + deps = [ + ":encryption", + "//base", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + "//crypto", + "//crypto:platform", + "//third_party/boringssl", + ] +} + +static_library("verification") { + sources = [ + "verification.cc", + "verification.h", + ] + deps = [ + "//base", + "//components/reporting/util:status", + "//third_party/boringssl", + ] +} + +static_library("test_support") { + testonly = true + sources = [ + "test_encryption_module.cc", + "test_encryption_module.h", + ] + deps = [ + ":decryption", + ":encryption", + ":encryption_module", + ":verification", + "//base", + "//base/test:test_support", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + "//components/reporting/util:status_macros", + "//testing/gmock", + "//testing/gtest", + "//third_party/boringssl:boringssl", + ] +} + +# All unit tests are built as part of the //components:components_unittests +# target and must be one targets named "unit_tests". +source_set("unit_tests") { + testonly = true + sources = [ + "encryption_module_unittest.cc", + "verification_unittest.cc", + ] + deps = [ + ":decryption", + ":encryption", + ":encryption_module", + ":test_support", + ":verification", + "//base", + "//base/test:test_support", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + "//components/reporting/util:status_macros", + "//testing/gmock", + "//testing/gtest", + "//third_party/boringssl:boringssl", + ] +} diff --git a/chromium/components/reporting/encryption/DEPS b/chromium/components/reporting/encryption/DEPS new file mode 100644 index 00000000000..f4cb4c0b915 --- /dev/null +++ b/chromium/components/reporting/encryption/DEPS @@ -0,0 +1,5 @@ +include_rules = [ + "+base", + "+crypto", + "+third_party/boringssl/src/include", +] diff --git a/chromium/components/reporting/encryption/decryption.cc b/chromium/components/reporting/encryption/decryption.cc new file mode 100644 index 00000000000..9efa787add5 --- /dev/null +++ b/chromium/components/reporting/encryption/decryption.cc @@ -0,0 +1,224 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/decryption.h" + +#include <limits> +#include <string> + +#include "base/containers/span.h" +#include "base/hash/hash.h" +#include "base/memory/ptr_util.h" +#include "base/rand_util.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_piece.h" +#include "base/task/post_task.h" +#include "base/task_runner.h" +#include "components/reporting/encryption/encryption.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" +#include "crypto/aead.h" +#include "crypto/openssl_util.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" +#include "third_party/boringssl/src/include/openssl/digest.h" +#include "third_party/boringssl/src/include/openssl/hkdf.h" + +namespace reporting { + +Decryptor::Handle::Handle(base::StringPiece shared_secret, + scoped_refptr<Decryptor> decryptor) + : shared_secret_(shared_secret), decryptor_(decryptor) {} + +Decryptor::Handle::~Handle() = default; + +void Decryptor::Handle::AddToRecord(base::StringPiece data, + base::OnceCallback<void(Status)> cb) { + // Add piece of data to the record. + record_.append(data.data(), data.size()); + std::move(cb).Run(Status::StatusOK()); +} + +void Decryptor::Handle::CloseRecord( + base::OnceCallback<void(StatusOr<base::StringPiece>)> cb) { + // Make sure the record self-destructs when returning from this method. + const auto self_destruct = base::WrapUnique(this); + + // Decrypt the data with symmetric key using AEAD interface. + crypto::Aead aead(crypto::Aead::CHACHA20_POLY1305); + + // Produce symmetric key from shared secret using HKDF. + // Since the original keys were only used once, no salt and context is needed. + const auto out_symmetric_key = std::make_unique<uint8_t[]>(aead.KeyLength()); + if (!HKDF(out_symmetric_key.get(), aead.KeyLength(), /*digest=*/EVP_sha256(), + reinterpret_cast<const uint8_t*>(shared_secret_.data()), + shared_secret_.size(), + /*salt=*/nullptr, /*salt_len=*/0, + /*info=*/nullptr, /*info_len=*/0)) { + std::move(cb).Run( + Status(error::INTERNAL, "Symmetric key extraction failed")); + return; + } + + // Use the symmetric key for data decryption. + aead.Init(base::make_span(out_symmetric_key.get(), aead.KeyLength())); + + // Set nonce to 0s, since a symmetric key is only used once. + // Note: if we ever start reusing the same symmetric key, we will need + // to generate new nonce for every record and transfer it to the peer. + std::string nonce(aead.NonceLength(), 0); + + // Decrypt collected record. + std::string decrypted; + if (!aead.Open(record_, nonce, std::string(), &decrypted)) { + std::move(cb).Run(Status(error::INTERNAL, "Failed to decrypt")); + return; + } + record_.clear(); // Free unused memory. + + // Return decrypted record. + std::move(cb).Run(decrypted); +} + +void Decryptor::OpenRecord(base::StringPiece shared_secret, + base::OnceCallback<void(StatusOr<Handle*>)> cb) { + std::move(cb).Run(new Handle(shared_secret, this)); +} + +StatusOr<std::string> Decryptor::DecryptSecret( + base::StringPiece private_key, + base::StringPiece peer_public_value) { + // Verify the keys. + if (private_key.size() != X25519_PRIVATE_KEY_LEN) { + return Status( + error::FAILED_PRECONDITION, + base::StrCat({"Private key size mismatch, expected=", + base::NumberToString(X25519_PRIVATE_KEY_LEN), + " actual=", base::NumberToString(private_key.size())})); + } + if (peer_public_value.size() != X25519_PUBLIC_VALUE_LEN) { + return Status( + error::FAILED_PRECONDITION, + base::StrCat({"Public key size mismatch, expected=", + base::NumberToString(X25519_PUBLIC_VALUE_LEN), " actual=", + base::NumberToString(peer_public_value.size())})); + } + + // Compute shared secret. + uint8_t out_shared_value[X25519_SHARED_KEY_LEN]; + if (!X25519(out_shared_value, + reinterpret_cast<const uint8_t*>(private_key.data()), + reinterpret_cast<const uint8_t*>(peer_public_value.data()))) { + return Status(error::DATA_LOSS, "Curve25519 decryption failed"); + } + + return std::string(reinterpret_cast<const char*>(out_shared_value), + X25519_SHARED_KEY_LEN); +} + +Decryptor::Decryptor() + : keys_sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::BEST_EFFORT, base::MayBlock()})) { + DETACH_FROM_SEQUENCE(keys_sequence_checker_); +} + +Decryptor::~Decryptor() = default; + +void Decryptor::RecordKeyPair( + base::StringPiece private_key, + base::StringPiece public_key, + base::OnceCallback<void(StatusOr<Encryptor::PublicKeyId>)> cb) { + // Schedule key recording on the sequenced task runner. + keys_sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce( + [](std::string public_key, KeyInfo key_info, + base::OnceCallback<void(StatusOr<Encryptor::PublicKeyId>)> cb, + scoped_refptr<Decryptor> decryptor) { + DCHECK_CALLED_ON_VALID_SEQUENCE(decryptor->keys_sequence_checker_); + StatusOr<Encryptor::PublicKeyId> result; + if (key_info.private_key.size() != X25519_PRIVATE_KEY_LEN) { + result = Status( + error::FAILED_PRECONDITION, + base::StrCat( + {"Private key size mismatch, expected=", + base::NumberToString(X25519_PRIVATE_KEY_LEN), " actual=", + base::NumberToString(key_info.private_key.size())})); + } else if (public_key.size() != X25519_PUBLIC_VALUE_LEN) { + result = Status( + error::FAILED_PRECONDITION, + base::StrCat( + {"Public key size mismatch, expected=", + base::NumberToString(X25519_PUBLIC_VALUE_LEN), + " actual=", base::NumberToString(public_key.size())})); + } else { + // Assign a random number to be public key id for testing purposes + // only (in production it will be retrieved from the server as + // 'int32'). + const Encryptor::PublicKeyId public_key_id = base::RandGenerator( + std::numeric_limits<Encryptor::PublicKeyId>::max()); + if (!decryptor->keys_.emplace(public_key_id, key_info).second) { + result = Status(error::ALREADY_EXISTS, + base::StrCat({"Public key='", public_key, + "' already recorded"})); + } else { + result = public_key_id; + } + } + // Schedule response on a generic thread pool. + base::ThreadPool::PostTask( + FROM_HERE, base::BindOnce( + [](base::OnceCallback<void( + StatusOr<Encryptor::PublicKeyId>)> cb, + StatusOr<Encryptor::PublicKeyId> result) { + std::move(cb).Run(result); + }, + std::move(cb), result)); + }, + std::string(public_key), + KeyInfo{.private_key = std::string(private_key), + .time_stamp = base::Time::Now()}, + std::move(cb), base::WrapRefCounted(this))); +} + +void Decryptor::RetrieveMatchingPrivateKey( + Encryptor::PublicKeyId public_key_id, + base::OnceCallback<void(StatusOr<std::string>)> cb) { + // Schedule key retrieval on the sequenced task runner. + keys_sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce( + [](Encryptor::PublicKeyId public_key_id, + base::OnceCallback<void(StatusOr<std::string>)> cb, + scoped_refptr<Decryptor> decryptor) { + DCHECK_CALLED_ON_VALID_SEQUENCE(decryptor->keys_sequence_checker_); + auto key_info_it = decryptor->keys_.find(public_key_id); + if (key_info_it != decryptor->keys_.end()) { + DCHECK_EQ(key_info_it->second.private_key.size(), + static_cast<size_t>(X25519_PRIVATE_KEY_LEN)); + } + // Schedule response on a generic thread pool. + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + [](base::OnceCallback<void(StatusOr<std::string>)> cb, + StatusOr<std::string> result) { + std::move(cb).Run(result); + }, + std::move(cb), + key_info_it == decryptor->keys_.end() + ? StatusOr<std::string>(Status( + error::NOT_FOUND, "Matching key not found")) + : key_info_it->second.private_key)); + }, + public_key_id, std::move(cb), base::WrapRefCounted(this))); +} + +StatusOr<scoped_refptr<Decryptor>> Decryptor::Create() { + // Make sure OpenSSL is initialized, in order to avoid data races later. + crypto::EnsureOpenSSLInit(); + return base::WrapRefCounted(new Decryptor()); +} + +} // namespace reporting diff --git a/chromium/components/reporting/encryption/decryption.h b/chromium/components/reporting/encryption/decryption.h new file mode 100644 index 00000000000..97c91c06bfb --- /dev/null +++ b/chromium/components/reporting/encryption/decryption.h @@ -0,0 +1,118 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_ENCRYPTION_DECRYPTION_H_ +#define COMPONENTS_REPORTING_ENCRYPTION_DECRYPTION_H_ + +#include <string> + +#include "base/callback.h" +#include "base/containers/flat_map.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "base/optional.h" +#include "base/strings/string_piece.h" +#include "base/threading/thread.h" +#include "base/threading/thread_task_runner_handle.h" +#include "components/reporting/encryption/encryption.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// Full implementation of Decryptor, intended for use in tests and potentially +// in reporting server (wrapped in a Java class). +// +// Curve25519 decryption of the symmetric key with asymmetric private key. +// ChaCha20_Poly1305 decryption and verification of a record in place with +// symmetric key. +// +// Instantiated by an implementation-specific factory: +// StatusOr<scoped_refptr<Decryptor>> Create(); +class Decryptor : public base::RefCountedThreadSafe<Decryptor> { + public: + // Decryption record handle, which is created by |OpenRecord| and can accept + // pieces of data to be decrypted as one record by calling |AddToRecord| + // multiple times. Resulting decrypted record is available once |CloseRecord| + // is called. + class Handle { + public: + Handle(base::StringPiece shared_secret, scoped_refptr<Decryptor> decryptor); + Handle(const Handle& other) = delete; + Handle& operator=(const Handle& other) = delete; + ~Handle(); + + // Adds piece of encrypted data to the record. + void AddToRecord(base::StringPiece data, + base::OnceCallback<void(Status)> cb); + + // Closes and attempts to decrypt the record. Hands over the decrypted data + // to be processed by the server (or Status if unsuccessful). Accesses key + // store to attempt all private keys that are considered to be valid, + // starting with the one that matches the hash. Self-destructs after the + // callback. + void CloseRecord(base::OnceCallback<void(StatusOr<base::StringPiece>)> cb); + + private: + // Shared secret based on which symmetric key is produced. + const std::string shared_secret_; + + // Accumulated data to decrypt. + std::string record_; + + scoped_refptr<Decryptor> decryptor_; + }; + + // Factory method to instantiate the Decryptor. + static StatusOr<scoped_refptr<Decryptor>> Create(); + + // Factory method creates a new record to collect data and decrypt them with + // the given encrypted key. Hands the handle raw pointer over to the callback, + // or error status. + void OpenRecord(base::StringPiece encrypted_key, + base::OnceCallback<void(StatusOr<Handle*>)> cb); + + // Recreates shared secret from local private key and peer public value and + // returns it or error status. + StatusOr<std::string> DecryptSecret(base::StringPiece public_key, + base::StringPiece peer_public_value); + + // Records a key pair (stores only private key). + // Executes on a sequenced thread, returns key id or error with callback. + void RecordKeyPair( + base::StringPiece private_key, + base::StringPiece public_key, + base::OnceCallback<void(StatusOr<Encryptor::PublicKeyId>)> cb); + + // Retrieves private key matching the public key hash. + // Executes on a sequenced thread, returns with callback. + void RetrieveMatchingPrivateKey( + Encryptor::PublicKeyId public_key_id, + base::OnceCallback<void(StatusOr<std::string>)> cb); + + private: + friend base::RefCountedThreadSafe<Decryptor>; + Decryptor(); + ~Decryptor(); + + // Map of hash(public_key)->{private key, time stamp} + // Private key is located by the hash of a public key, sent together with the + // encrypted record. Keys older than pre-defined threshold are discarded. + // Time stamp allows to drop outdated keys (not implemented yet). + struct KeyInfo { + std::string private_key; + base::Time time_stamp; + }; + base::flat_map<Encryptor::PublicKeyId, KeyInfo> keys_; + + // Sequential task runner for all keys_ activities: + // recording, lookup, purge. + scoped_refptr<base::SequencedTaskRunner> keys_sequenced_task_runner_; + + SEQUENCE_CHECKER(keys_sequence_checker_); +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_ENCRYPTION_DECRYPTION_H_ diff --git a/chromium/components/reporting/encryption/encryption.cc b/chromium/components/reporting/encryption/encryption.cc new file mode 100644 index 00000000000..e880dcc5408 --- /dev/null +++ b/chromium/components/reporting/encryption/encryption.cc @@ -0,0 +1,201 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/encryption.h" + +#include <string> +#include <utility> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/containers/span.h" +#include "base/hash/hash.h" +#include "base/memory/ptr_util.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/task/post_task.h" +#include "base/task_runner.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" +#include "crypto/aead.h" +#include "crypto/openssl_util.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" +#include "third_party/boringssl/src/include/openssl/digest.h" +#include "third_party/boringssl/src/include/openssl/hkdf.h" + +namespace reporting { + +Encryptor::Handle::Handle(scoped_refptr<Encryptor> encryptor) + : encryptor_(encryptor) {} + +Encryptor::Handle::~Handle() = default; + +void Encryptor::Handle::AddToRecord(base::StringPiece data, + base::OnceCallback<void(Status)> cb) { + // Append new data to the record. + record_.append(data.data(), data.size()); + std::move(cb).Run(Status::StatusOK()); +} + +void Encryptor::Handle::CloseRecord( + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb) { + // Retrieves asymmetric public key to use. + encryptor_->RetrieveAsymmetricKey(base::BindOnce( + &Handle::ProduceEncryptedRecord, base::Unretained(this), std::move(cb))); +} + +void Encryptor::Handle::ProduceEncryptedRecord( + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb, + StatusOr<std::pair<std::string, PublicKeyId>> asymmetric_key_result) { + // Make sure the record self-destructs when returning from this method. + const auto self_destruct = base::WrapUnique(this); + + // Validate keys. + if (!asymmetric_key_result.ok()) { + std::move(cb).Run(asymmetric_key_result.status()); + return; + } + const auto& asymmetric_key = asymmetric_key_result.ValueOrDie(); + if (asymmetric_key.first.size() != X25519_PUBLIC_VALUE_LEN) { + std::move(cb).Run(Status( + error::INTERNAL, + base::StrCat({"Asymmetric key size mismatch, expected=", + base::NumberToString(X25519_PUBLIC_VALUE_LEN), " actual=", + base::NumberToString(asymmetric_key.first.size())}))); + return; + } + + // Generate new pair of private key and public value. + uint8_t out_public_value[X25519_PUBLIC_VALUE_LEN]; + uint8_t out_private_key[X25519_PRIVATE_KEY_LEN]; + X25519_keypair(out_public_value, out_private_key); + + // Compute shared secret. + uint8_t out_shared_secret[X25519_SHARED_KEY_LEN]; + if (!X25519(out_shared_secret, out_private_key, + reinterpret_cast<const uint8_t*>(asymmetric_key.first.data()))) { + std::move(cb).Run(Status(error::DATA_LOSS, "Curve25519 encryption failed")); + return; + } + + // Encrypt the data with symmetric key using AEAD interface. + crypto::Aead aead(crypto::Aead::CHACHA20_POLY1305); + + // Produce symmetric key from shared secret using HKDF. + // Since the keys above are only used once, no salt and context is provided. + const auto out_symmetric_key = std::make_unique<uint8_t[]>(aead.KeyLength()); + if (!HKDF(out_symmetric_key.get(), aead.KeyLength(), /*digest=*/EVP_sha256(), + out_shared_secret, X25519_SHARED_KEY_LEN, + /*salt=*/nullptr, /*salt_len=*/0, + /*info=*/nullptr, /*info_len=*/0)) { + std::move(cb).Run( + Status(error::INTERNAL, "Symmetric key extraction failed")); + return; + } + + // Use the symmetric key for data encryption. + aead.Init(base::make_span(out_symmetric_key.get(), aead.KeyLength())); + + // Set nonce to 0s, since a symmetric key is only used once. + // Note: if we ever start reusing the same symmetric key, we will need + // to generate new nonce for every record and transfer it to the peer. + std::string nonce(aead.NonceLength(), 0); + + // Prepare encrypted record. + EncryptedRecord encrypted_record; + encrypted_record.mutable_encryption_info()->set_public_key_id( + asymmetric_key.second); + encrypted_record.mutable_encryption_info()->set_encryption_key( + reinterpret_cast<const char*>(out_public_value), X25519_PUBLIC_VALUE_LEN); + + // Encrypt the whole record. + if (!aead.Seal(record_, nonce, std::string(), + encrypted_record.mutable_encrypted_wrapped_record()) || + encrypted_record.encrypted_wrapped_record().empty()) { + std::move(cb).Run(Status(error::INTERNAL, "Failed to encrypt the record")); + return; + } + record_.clear(); // Free unused memory. + + // Return EncryptedRecord. + std::move(cb).Run(encrypted_record); +} + +Encryptor::Encryptor() + : asymmetric_key_sequenced_task_runner_( + base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::BEST_EFFORT, base::MayBlock()})) { + DETACH_FROM_SEQUENCE(asymmetric_key_sequence_checker_); +} + +Encryptor::~Encryptor() = default; + +void Encryptor::UpdateAsymmetricKey( + base::StringPiece new_public_key, + PublicKeyId new_public_key_id, + base::OnceCallback<void(Status)> response_cb) { + if (new_public_key.empty()) { + std::move(response_cb) + .Run(Status(error::INVALID_ARGUMENT, "Provided key is empty")); + return; + } + + // Schedule key update on the sequenced task runner. + asymmetric_key_sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce( + [](base::StringPiece new_public_key, PublicKeyId new_public_key_id, + scoped_refptr<Encryptor> encryptor) { + encryptor->asymmetric_key_ = + std::make_pair(std::string(new_public_key), new_public_key_id); + }, + std::string(new_public_key), new_public_key_id, + base::WrapRefCounted(this))); + + // Response OK not waiting for the update. + std::move(response_cb).Run(Status::StatusOK()); +} + +void Encryptor::OpenRecord(base::OnceCallback<void(StatusOr<Handle*>)> cb) { + std::move(cb).Run(new Handle(this)); +} + +void Encryptor::RetrieveAsymmetricKey( + base::OnceCallback<void(StatusOr<std::pair<std::string, PublicKeyId>>)> + cb) { + // Schedule key retrieval on the sequenced task runner. + asymmetric_key_sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce( + [](base::OnceCallback<void( + StatusOr<std::pair<std::string, PublicKeyId>>)> cb, + scoped_refptr<Encryptor> encryptor) { + DCHECK_CALLED_ON_VALID_SEQUENCE( + encryptor->asymmetric_key_sequence_checker_); + StatusOr<std::pair<std::string, PublicKeyId>> response; + // Schedule response on regular thread pool. + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + [](base::OnceCallback<void( + StatusOr<std::pair<std::string, PublicKeyId>>)> cb, + StatusOr<std::pair<std::string, PublicKeyId>> response) { + std::move(cb).Run(response); + }, + std::move(cb), + !encryptor->asymmetric_key_.has_value() + ? StatusOr<std::pair<std::string, PublicKeyId>>(Status( + error::NOT_FOUND, "Asymmetric key not set")) + : encryptor->asymmetric_key_.value())); + }, + std::move(cb), base::WrapRefCounted(this))); +} + +StatusOr<scoped_refptr<Encryptor>> Encryptor::Create() { + // Make sure OpenSSL is initialized, in order to avoid data races later. + crypto::EnsureOpenSSLInit(); + return base::WrapRefCounted(new Encryptor()); +} + +} // namespace reporting diff --git a/chromium/components/reporting/encryption/encryption.h b/chromium/components/reporting/encryption/encryption.h new file mode 100644 index 00000000000..eea82c3cdda --- /dev/null +++ b/chromium/components/reporting/encryption/encryption.h @@ -0,0 +1,117 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_ENCRYPTION_ENCRYPTION_H_ +#define COMPONENTS_REPORTING_ENCRYPTION_ENCRYPTION_H_ + +#include <string> +#include <utility> + +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "base/optional.h" +#include "base/strings/string_piece.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// Full implementation of Encryptor, intended for use in reporting client. +// ChaCha20_Poly1305 AEAD encryption of a record in place with symmetric key. +// Curve25519 encryption of the symmetric key with asymmetric public key. +// +// We generate new Curve25519 public/private keys pair for each record. +// Then we produce Curve25519 shared secret from our private key and peer's +// public key, and use it for ChaCha20_Poly1305 AEAD encryption of the record. +// We send out our public value (calling it encrypted symmetric key) together +// with encrypted record. +// +// Upon receiving the encrypted message the peer will produce the same shared +// secret by combining their private key and our public key, and use it as +// a symmetric key for ChaCha20_Poly1305 decryption and validation of the +// record. +// +// Instantiated by a factory: +// StatusOr<scoped_refptr<Encryptor>> Create(); +// The implementation class should never be used directly by the client code. +class Encryptor : public base::RefCountedThreadSafe<Encryptor> { + public: + // Public key id, as defined by Keystore. + using PublicKeyId = int32_t; + + // Encryption record handle, which is created by |OpenRecord| and can accept + // pieces of data to be encrypted as one record by calling |AddToRecord| + // multiple times. Resulting encrypted record is available once |CloseRecord| + // is called. + class Handle { + public: + explicit Handle(scoped_refptr<Encryptor> encryptor); + Handle(const Handle& other) = delete; + Handle& operator=(const Handle& other) = delete; + ~Handle(); + + // Adds piece of data to the record. + void AddToRecord(base::StringPiece data, + base::OnceCallback<void(Status)> cb); + + // Closes and encrypts the record, hands over the data (encrypted with + // symmetric key) and the key (encrypted with asymmetric key) to be recorded + // by the client (or Status if unsuccessful). Self-destructs after the + // callback. + void CloseRecord(base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb); + + private: + // Helper method to compose EncryptedRecord. Called by |CloseRecord| + // as a callback after asynchronous retrieval of the asymmetric key. + void ProduceEncryptedRecord( + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb, + StatusOr<std::pair<std::string, PublicKeyId>> asymmetric_key_result); + + // Accumulated data to encrypt. + std::string record_; + + scoped_refptr<Encryptor> encryptor_; + }; + + // Factory method to instantiate the Encryptor. + static StatusOr<scoped_refptr<Encryptor>> Create(); + + // Factory method creates new record to collect data and encrypt them. + // Hands the Handle raw pointer over to the callback, or error status). + void OpenRecord(base::OnceCallback<void(StatusOr<Handle*>)> cb); + + // Delivers public asymmetric key and its id to the implementation. + // To affect specific record, must happen before Handle::CloseRecord + // (it is OK to do it after OpenRecord and Handle::AddToRecord). + // Executes on a sequenced thread, returns with callback. + void UpdateAsymmetricKey(base::StringPiece new_public_key, + PublicKeyId new_public_key_id, + base::OnceCallback<void(Status)> response_cb); + + // Retrieves the current public key. + // Executes on a sequenced thread, returns with callback. + void RetrieveAsymmetricKey( + base::OnceCallback<void(StatusOr<std::pair<std::string, PublicKeyId>>)> + cb); + + private: + friend class base::RefCountedThreadSafe<Encryptor>; + Encryptor(); + ~Encryptor(); + + // Public key used for asymmetric encryption of symmetric key and its id. + base::Optional<std::pair<std::string, PublicKeyId>> asymmetric_key_; + + // Sequential task runner for all asymmetric_key_ activities: update, read. + scoped_refptr<base::SequencedTaskRunner> + asymmetric_key_sequenced_task_runner_; + + SEQUENCE_CHECKER(asymmetric_key_sequence_checker_); +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_ENCRYPTION_ENCRYPTION_H_ diff --git a/chromium/components/reporting/encryption/encryption_module.cc b/chromium/components/reporting/encryption/encryption_module.cc new file mode 100644 index 00000000000..b9538e23ad3 --- /dev/null +++ b/chromium/components/reporting/encryption/encryption_module.cc @@ -0,0 +1,134 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/encryption_module.h" + +#include <atomic> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/feature_list.h" +#include "base/strings/string_piece.h" +#include "base/task/thread_pool.h" +#include "base/time/time.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +namespace { + +// Temporary: enable/disable encryption. +const base::Feature kEncryptedReportingFeature{ + EncryptionModule::kEncryptedReporting, base::FEATURE_DISABLED_BY_DEFAULT}; + +// Helper function for asynchronous encryption. +void AddToRecord(base::StringPiece record, + Encryptor::Handle* handle, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb) { + handle->AddToRecord( + record, + base::BindOnce( + [](Encryptor::Handle* handle, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb, + Status status) { + if (!status.ok()) { + std::move(cb).Run(status); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&Encryptor::Handle::CloseRecord, + base::Unretained(handle), std::move(cb))); + }, + base::Unretained(handle), std::move(cb))); +} + +} // namespace + +// static +const char EncryptionModule::kEncryptedReporting[] = "EncryptedReporting"; + +// static +bool EncryptionModule::is_enabled() { + return base::FeatureList::IsEnabled(kEncryptedReportingFeature); +} + +EncryptionModule::EncryptionModule(base::TimeDelta renew_encryption_key_period) + : renew_encryption_key_period_(renew_encryption_key_period) { + auto encryptor_result = Encryptor::Create(); + DCHECK(encryptor_result.ok()); + encryptor_ = std::move(encryptor_result.ValueOrDie()); +} + +EncryptionModule::~EncryptionModule() = default; + +void EncryptionModule::EncryptRecord( + base::StringPiece record, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb) const { + if (!is_enabled()) { + // Encryptor disabled. + EncryptedRecord encrypted_record; + encrypted_record.mutable_encrypted_wrapped_record()->assign(record.begin(), + record.end()); + // encryption_info is not set. + std::move(cb).Run(std::move(encrypted_record)); + return; + } + + // Encryptor enabled: start encryption of the record as a whole. + if (!has_encryption_key()) { + // Encryption key is not available. + std::move(cb).Run( + Status(error::NOT_FOUND, "Cannot encrypt record - no key")); + return; + } + // Encryption key is available, encrypt. + encryptor_->OpenRecord(base::BindOnce( + [](base::StringPiece record, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb, + StatusOr<Encryptor::Handle*> handle_result) { + if (!handle_result.ok()) { + std::move(cb).Run(handle_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&AddToRecord, std::string(record), + base::Unretained(handle_result.ValueOrDie()), + std::move(cb))); + }, + std::string(record), std::move(cb))); +} + +void EncryptionModule::UpdateAsymmetricKey( + base::StringPiece new_public_key, + Encryptor::PublicKeyId new_public_key_id, + base::OnceCallback<void(Status)> response_cb) { + encryptor_->UpdateAsymmetricKey( + new_public_key, new_public_key_id, + base::BindOnce( + [](EncryptionModule* encryption_module, + base::OnceCallback<void(Status)> response_cb, Status status) { + if (status.ok()) { + encryption_module->last_encryption_key_update_.store( + base::TimeTicks::Now()); + } + std::move(response_cb).Run(status); + }, + base::Unretained(this), std::move(response_cb))); +} + +bool EncryptionModule::has_encryption_key() const { + return !last_encryption_key_update_.load().is_null(); +} + +bool EncryptionModule::need_encryption_key() const { + return !has_encryption_key() || + last_encryption_key_update_.load() + renew_encryption_key_period_ < + base::TimeTicks::Now(); +} + +} // namespace reporting diff --git a/chromium/components/reporting/encryption/encryption_module.h b/chromium/components/reporting/encryption/encryption_module.h new file mode 100644 index 00000000000..59aa58fdbc7 --- /dev/null +++ b/chromium/components/reporting/encryption/encryption_module.h @@ -0,0 +1,80 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_ENCRYPTION_ENCRYPTION_MODULE_H_ +#define COMPONENTS_REPORTING_ENCRYPTION_ENCRYPTION_MODULE_H_ + +#include <atomic> + +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/strings/string_piece.h" +#include "base/time/time.h" +#include "components/reporting/encryption/encryption.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +class EncryptionModule : public base::RefCountedThreadSafe<EncryptionModule> { + public: + // Feature to enable/disable encryption. + // By default encryption is disabled, until server can support decryption. + static const char kEncryptedReporting[]; + + explicit EncryptionModule(base::TimeDelta renew_encryption_key_period = + base::TimeDelta::FromDays(1)); + EncryptionModule(const EncryptionModule& other) = delete; + EncryptionModule& operator=(const EncryptionModule& other) = delete; + + // EncryptRecord will attempt to encrypt the provided |record| and respond + // with the callback. On success the returned EncryptedRecord will contain + // the encrypted string and encryption information. EncryptedRecord then can + // be further updated by the caller. + virtual void EncryptRecord( + base::StringPiece record, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb) const; + + // Records current public asymmetric key. + virtual void UpdateAsymmetricKey( + base::StringPiece new_public_key, + Encryptor::PublicKeyId new_public_key_id, + base::OnceCallback<void(Status)> response_cb); + + // Returns `false` if encryption key has not been set yet, and `true` + // otherwise. The result is lazy: the method may return `false` for some time + // even after the key has already been set - this is harmless, since resetting + // or even changing the key is OK at any time. + bool has_encryption_key() const; + + // Returns `true` if encryption key has not been set yet or it is too old + // (received more than |renew_encryption_key_period| ago). + bool need_encryption_key() const; + + // Returns 'true' if |kEncryptedReporting| feature is enabled. + // To be removed once encryption becomes mandatory. + static bool is_enabled(); + + protected: + virtual ~EncryptionModule(); + + private: + friend base::RefCountedThreadSafe<EncryptionModule>; + + // Timestamp of the last public asymmetric key update by + // |UpdateAsymmetricKey|. Initial value base::TimeTicks() indicates key is not + // set yet. + std::atomic<base::TimeTicks> last_encryption_key_update_{base::TimeTicks()}; + + // Period of encryption key update. + const base::TimeDelta renew_encryption_key_period_; + + // Encryptor. + scoped_refptr<Encryptor> encryptor_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_ENCRYPTION_ENCRYPTION_MODULE_H_ diff --git a/chromium/components/reporting/encryption/encryption_module_unittest.cc b/chromium/components/reporting/encryption/encryption_module_unittest.cc new file mode 100644 index 00000000000..d5c95e92f19 --- /dev/null +++ b/chromium/components/reporting/encryption/encryption_module_unittest.cc @@ -0,0 +1,592 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/encryption_module.h" + +#include "base/bind.h" +#include "base/containers/flat_map.h" +#include "base/hash/hash.h" +#include "base/rand_util.h" +#include "base/strings/strcat.h" +#include "base/synchronization/waitable_event.h" +#include "base/test/scoped_feature_list.h" +#include "base/test/task_environment.h" +#include "base/time/time.h" +#include "components/reporting/encryption/decryption.h" +#include "components/reporting/encryption/encryption.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/status_macros.h" +#include "components/reporting/util/statusor.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" + +namespace reporting { +namespace { + +// Usage (in tests only): +// +// TestEvent<ResType> e; +// ... Do some async work passing e.cb() as a completion callback of +// base::OnceCallback<void(ResType* res)> type which also may perform some +// other action specified by |done| callback provided by the caller. +// ... = e.result(); // Will wait for e.cb() to be called and return the +// collected result. +// +template <typename ResType> +class TestEvent { + public: + TestEvent() : run_loop_(std::make_unique<base::RunLoop>()) {} + ~TestEvent() { EXPECT_FALSE(run_loop_->running()) << "Not responded"; } + TestEvent(const TestEvent& other) = delete; + TestEvent& operator=(const TestEvent& other) = delete; + ResType result() { + run_loop_->Run(); + return std::forward<ResType>(result_); + } + + // Completion callback to hand over to the processing method. + base::OnceCallback<void(ResType res)> cb() { + return base::BindOnce( + [](base::RunLoop* run_loop, ResType* result, ResType res) { + *result = std::forward<ResType>(res); + run_loop->Quit(); + }, + base::Unretained(run_loop_.get()), base::Unretained(&result_)); + } + + private: + std::unique_ptr<base::RunLoop> run_loop_; + ResType result_; +}; + +class EncryptionModuleTest : public ::testing::Test { + protected: + EncryptionModuleTest() = default; + + void SetUp() override { + // Enable encryption. + scoped_feature_list_.InitFromCommandLine( + {EncryptionModule::kEncryptedReporting}, {}); + + encryption_module_ = base::MakeRefCounted<EncryptionModule>(); + + auto decryptor_result = Decryptor::Create(); + ASSERT_OK(decryptor_result.status()) << decryptor_result.status(); + decryptor_ = std::move(decryptor_result.ValueOrDie()); + } + + StatusOr<EncryptedRecord> EncryptSync(base::StringPiece data) { + TestEvent<StatusOr<EncryptedRecord>> encrypt_record; + encryption_module_->EncryptRecord(data, encrypt_record.cb()); + return encrypt_record.result(); + } + + StatusOr<std::string> DecryptSync( + std::pair<std::string /*shared_secret*/, std::string /*encrypted_data*/> + encrypted) { + TestEvent<StatusOr<Decryptor::Handle*>> open_decrypt; + decryptor_->OpenRecord(encrypted.first, open_decrypt.cb()); + auto open_decrypt_result = open_decrypt.result(); + RETURN_IF_ERROR(open_decrypt_result.status()); + Decryptor::Handle* const dec_handle = open_decrypt_result.ValueOrDie(); + + TestEvent<Status> add_decrypt; + dec_handle->AddToRecord(encrypted.second, add_decrypt.cb()); + RETURN_IF_ERROR(add_decrypt.result()); + + std::string decrypted_string; + TestEvent<Status> close_decrypt; + dec_handle->CloseRecord(base::BindOnce( + [](std::string* decrypted_string, + base::OnceCallback<void(Status)> close_cb, + StatusOr<base::StringPiece> result) { + if (!result.ok()) { + std::move(close_cb).Run(result.status()); + return; + } + *decrypted_string = std::string(result.ValueOrDie()); + std::move(close_cb).Run(Status::StatusOK()); + }, + base::Unretained(&decrypted_string), close_decrypt.cb())); + RETURN_IF_ERROR(close_decrypt.result()); + return decrypted_string; + } + + StatusOr<std::string> DecryptMatchingSecret( + Encryptor::PublicKeyId public_key_id, + base::StringPiece encrypted_key) { + // Retrieve private key that matches public key hash. + TestEvent<StatusOr<std::string>> retrieve_private_key; + decryptor_->RetrieveMatchingPrivateKey(public_key_id, + retrieve_private_key.cb()); + ASSIGN_OR_RETURN(std::string private_key, retrieve_private_key.result()); + // Decrypt symmetric key with that private key and peer public key. + ASSIGN_OR_RETURN(std::string shared_secret, + decryptor_->DecryptSecret(private_key, encrypted_key)); + return shared_secret; + } + + Status AddNewKeyPair() { + // Generate new pair of private key and public value. + uint8_t out_public_value[X25519_PUBLIC_VALUE_LEN]; + uint8_t out_private_key[X25519_PRIVATE_KEY_LEN]; + X25519_keypair(out_public_value, out_private_key); + + TestEvent<StatusOr<Encryptor::PublicKeyId>> record_keys; + decryptor_->RecordKeyPair( + std::string(reinterpret_cast<const char*>(out_private_key), + X25519_PRIVATE_KEY_LEN), + std::string(reinterpret_cast<const char*>(out_public_value), + X25519_PUBLIC_VALUE_LEN), + record_keys.cb()); + ASSIGN_OR_RETURN(Encryptor::PublicKeyId new_public_key_id, + record_keys.result()); + TestEvent<Status> set_public_key; + encryption_module_->UpdateAsymmetricKey( + std::string(reinterpret_cast<const char*>(out_public_value), + X25519_PUBLIC_VALUE_LEN), + new_public_key_id, set_public_key.cb()); + RETURN_IF_ERROR(set_public_key.result()); + return Status::StatusOK(); + } + + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + + scoped_refptr<EncryptionModule> encryption_module_; + scoped_refptr<Decryptor> decryptor_; + + private: + base::test::ScopedFeatureList scoped_feature_list_; +}; + +TEST_F(EncryptionModuleTest, EncryptAndDecrypt) { + constexpr char kTestString[] = "ABCDEF"; + + // Register new pair of private key and public value. + ASSERT_OK(AddNewKeyPair()); + + // Encrypt the test string using the last public value. + const auto encrypted_result = EncryptSync(kTestString); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + + // Decrypt shared secret with private asymmetric key. + auto decrypt_secret_result = DecryptMatchingSecret( + encrypted_result.ValueOrDie().encryption_info().public_key_id(), + encrypted_result.ValueOrDie().encryption_info().encryption_key()); + ASSERT_OK(decrypt_secret_result.status()) << decrypt_secret_result.status(); + + // Decrypt back. + const auto decrypted_result = DecryptSync( + std::make_pair(decrypt_secret_result.ValueOrDie(), + encrypted_result.ValueOrDie().encrypted_wrapped_record())); + ASSERT_OK(decrypted_result.status()) << decrypted_result.status(); + + EXPECT_THAT(decrypted_result.ValueOrDie(), ::testing::StrEq(kTestString)); +} + +TEST_F(EncryptionModuleTest, EncryptionDisabled) { + constexpr char kTestString[] = "ABCDEF"; + + // Disable encryption. + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitFromCommandLine( + {}, {EncryptionModule::kEncryptedReporting}); + + // Encrypt the test string. + const auto encrypted_result = EncryptSync(kTestString); + ASSERT_OK(encrypted_result.status()); + + // Expect the result to be identical to the original record, + // and have no encryption_info. + EXPECT_EQ(encrypted_result.ValueOrDie().encrypted_wrapped_record(), + kTestString); + EXPECT_FALSE(encrypted_result.ValueOrDie().has_encryption_info()); +} + +TEST_F(EncryptionModuleTest, PublicKeyUpdate) { + constexpr char kTestString[] = "ABCDEF"; + + // No key yet, attempt to encrypt the test string. + ASSERT_FALSE(encryption_module_->has_encryption_key()); + ASSERT_TRUE(encryption_module_->need_encryption_key()); + auto encrypted_result = EncryptSync(kTestString); + EXPECT_EQ(encrypted_result.status().error_code(), error::NOT_FOUND); + + // Register new pair of private key and public value. + ASSERT_OK(AddNewKeyPair()); + ASSERT_TRUE(encryption_module_->has_encryption_key()); + ASSERT_FALSE(encryption_module_->need_encryption_key()); + + // Encrypt the test string using the last public value. + encrypted_result = EncryptSync(kTestString); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + + // Simulate short wait. Key is still available and not needed. + task_environment_.FastForwardBy(base::TimeDelta::FromHours(8)); + ASSERT_TRUE(encryption_module_->has_encryption_key()); + ASSERT_FALSE(encryption_module_->need_encryption_key()); + encrypted_result = EncryptSync(kTestString); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + + // Simulate long wait. Key is still available, but is needed now. + task_environment_.FastForwardBy(base::TimeDelta::FromDays(1)); + ASSERT_TRUE(encryption_module_->has_encryption_key()); + ASSERT_TRUE(encryption_module_->need_encryption_key()); + encrypted_result = EncryptSync(kTestString); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + + // Register one more pair of private key and public value. + ASSERT_OK(AddNewKeyPair()); + ASSERT_TRUE(encryption_module_->has_encryption_key()); + ASSERT_FALSE(encryption_module_->need_encryption_key()); + encrypted_result = EncryptSync(kTestString); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); +} + +TEST_F(EncryptionModuleTest, EncryptAndDecryptMultiple) { + constexpr const char* kTestStrings[] = {"Rec1", "Rec22", "Rec333", + "Rec4444", "Rec55555", "Rec666666"}; + // Encrypted records. + std::vector<EncryptedRecord> encrypted_records; + + // 1. Register first key pair. + ASSERT_OK(AddNewKeyPair()); + + // 2. Encrypt 3 test strings. + for (const char* test_string : + {kTestStrings[0], kTestStrings[1], kTestStrings[2]}) { + const auto encrypted_result = EncryptSync(test_string); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + encrypted_records.emplace_back(encrypted_result.ValueOrDie()); + } + + // 3. Register second key pair. + ASSERT_OK(AddNewKeyPair()); + + // 4. Encrypt 2 test strings. + for (const char* test_string : {kTestStrings[3], kTestStrings[4]}) { + const auto encrypted_result = EncryptSync(test_string); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + encrypted_records.emplace_back(encrypted_result.ValueOrDie()); + } + + // 3. Register third key pair. + ASSERT_OK(AddNewKeyPair()); + + // 4. Encrypt one more test strings. + for (const char* test_string : {kTestStrings[5]}) { + const auto encrypted_result = EncryptSync(test_string); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + encrypted_records.emplace_back(encrypted_result.ValueOrDie()); + } + + // For every encrypted record: + for (size_t i = 0; i < encrypted_records.size(); ++i) { + // Decrypt encrypted_key with private asymmetric key. + auto decrypt_secret_result = DecryptMatchingSecret( + encrypted_records[i].encryption_info().public_key_id(), + encrypted_records[i].encryption_info().encryption_key()); + ASSERT_OK(decrypt_secret_result.status()) << decrypt_secret_result.status(); + + // Decrypt back. + const auto decrypted_result = DecryptSync( + std::make_pair(decrypt_secret_result.ValueOrDie(), + encrypted_records[i].encrypted_wrapped_record())); + ASSERT_OK(decrypted_result.status()) << decrypted_result.status(); + + // Verify match. + EXPECT_THAT(decrypted_result.ValueOrDie(), + ::testing::StrEq(kTestStrings[i])); + } +} + +TEST_F(EncryptionModuleTest, EncryptAndDecryptMultipleParallel) { + // Context of single encryption. Self-destructs upon completion or failure. + class SingleEncryptionContext { + public: + SingleEncryptionContext( + base::StringPiece test_string, + base::StringPiece public_key, + Encryptor::PublicKeyId public_key_id, + scoped_refptr<EncryptionModule> encryption_module, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> response) + : test_string_(test_string), + public_key_(public_key), + public_key_id_(public_key_id), + encryption_module_(encryption_module), + response_(std::move(response)) {} + + SingleEncryptionContext(const SingleEncryptionContext& other) = delete; + SingleEncryptionContext& operator=(const SingleEncryptionContext& other) = + delete; + + ~SingleEncryptionContext() { + DCHECK(!response_) << "Self-destruct without prior response"; + } + + void Start() { + base::ThreadPool::PostTask( + FROM_HERE, base::BindOnce(&SingleEncryptionContext::SetPublicKey, + base::Unretained(this))); + } + + private: + void Respond(StatusOr<EncryptedRecord> result) { + std::move(response_).Run(result); + delete this; + } + void SetPublicKey() { + encryption_module_->UpdateAsymmetricKey( + public_key_, public_key_id_, + base::BindOnce( + [](SingleEncryptionContext* self, Status status) { + if (!status.ok()) { + self->Respond(status); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleEncryptionContext::EncryptRecord, + base::Unretained(self))); + }, + base::Unretained(this))); + } + void EncryptRecord() { + encryption_module_->EncryptRecord( + test_string_, + base::BindOnce( + [](SingleEncryptionContext* self, + StatusOr<EncryptedRecord> encryption_result) { + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleEncryptionContext::Respond, + base::Unretained(self), encryption_result)); + }, + base::Unretained(this))); + } + + private: + const std::string test_string_; + const std::string public_key_; + const Encryptor::PublicKeyId public_key_id_; + const scoped_refptr<EncryptionModule> encryption_module_; + base::OnceCallback<void(StatusOr<EncryptedRecord>)> response_; + }; + + // Context of single decryption. Self-destructs upon completion or failure. + class SingleDecryptionContext { + public: + SingleDecryptionContext( + const EncryptedRecord& encrypted_record, + scoped_refptr<Decryptor> decryptor, + base::OnceCallback<void(StatusOr<base::StringPiece>)> response) + : encrypted_record_(encrypted_record), + decryptor_(decryptor), + response_(std::move(response)) {} + + SingleDecryptionContext(const SingleDecryptionContext& other) = delete; + SingleDecryptionContext& operator=(const SingleDecryptionContext& other) = + delete; + + ~SingleDecryptionContext() { + DCHECK(!response_) << "Self-destruct without prior response"; + } + + void Start() { + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::RetrieveMatchingPrivateKey, + base::Unretained(this))); + } + + private: + void Respond(StatusOr<base::StringPiece> result) { + std::move(response_).Run(result); + delete this; + } + + void RetrieveMatchingPrivateKey() { + // Retrieve private key that matches public key hash. + decryptor_->RetrieveMatchingPrivateKey( + encrypted_record_.encryption_info().public_key_id(), + base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<std::string> private_key_result) { + if (!private_key_result.ok()) { + self->Respond(private_key_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + &SingleDecryptionContext::DecryptSharedSecret, + base::Unretained(self), + private_key_result.ValueOrDie())); + }, + base::Unretained(this))); + } + + void DecryptSharedSecret(base::StringPiece private_key) { + // Decrypt shared secret from private key and peer public key. + auto shared_secret_result = decryptor_->DecryptSecret( + private_key, encrypted_record_.encryption_info().encryption_key()); + if (!shared_secret_result.ok()) { + Respond(shared_secret_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, base::BindOnce(&SingleDecryptionContext::OpenRecord, + base::Unretained(this), + shared_secret_result.ValueOrDie())); + } + + void OpenRecord(base::StringPiece shared_secret) { + decryptor_->OpenRecord( + shared_secret, + base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<Decryptor::Handle*> handle_result) { + if (!handle_result.ok()) { + self->Respond(handle_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + &SingleDecryptionContext::AddToRecord, + base::Unretained(self), + base::Unretained(handle_result.ValueOrDie()))); + }, + base::Unretained(this))); + } + + void AddToRecord(Decryptor::Handle* handle) { + handle->AddToRecord( + encrypted_record_.encrypted_wrapped_record(), + base::BindOnce( + [](SingleDecryptionContext* self, Decryptor::Handle* handle, + Status status) { + if (!status.ok()) { + self->Respond(status); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::CloseRecord, + base::Unretained(self), + base::Unretained(handle))); + }, + base::Unretained(this), base::Unretained(handle))); + } + + void CloseRecord(Decryptor::Handle* handle) { + handle->CloseRecord(base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<base::StringPiece> decryption_result) { + self->Respond(decryption_result); + }, + base::Unretained(this))); + } + + private: + const EncryptedRecord encrypted_record_; + const scoped_refptr<Decryptor> decryptor_; + base::OnceCallback<void(StatusOr<base::StringPiece>)> response_; + }; + + constexpr std::array<const char*, 6> kTestStrings = { + "Rec1", "Rec22", "Rec333", "Rec4444", "Rec55555", "Rec666666"}; + + // Public and private key pairs in this test are reversed strings. + std::vector<std::string> private_key_strings; + std::vector<std::string> public_value_strings; + std::vector<Encryptor::PublicKeyId> public_value_ids; + for (size_t i = 0; i < 3; ++i) { + // Generate new pair of private key and public value. + uint8_t out_public_value[X25519_PUBLIC_VALUE_LEN]; + uint8_t out_private_key[X25519_PRIVATE_KEY_LEN]; + X25519_keypair(out_public_value, out_private_key); + private_key_strings.emplace_back( + reinterpret_cast<const char*>(out_private_key), X25519_PRIVATE_KEY_LEN); + public_value_strings.emplace_back( + reinterpret_cast<const char*>(out_public_value), + X25519_PUBLIC_VALUE_LEN); + } + + // Register all key pairs for decryption. + std::vector<TestEvent<StatusOr<Encryptor::PublicKeyId>>> record_results( + public_value_strings.size()); + for (size_t i = 0; i < public_value_strings.size(); ++i) { + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + [](base::StringPiece private_key_string, + base::StringPiece public_key_string, + scoped_refptr<Decryptor> decryptor, + base::OnceCallback<void(StatusOr<Encryptor::PublicKeyId>)> + done_cb) { + decryptor->RecordKeyPair(private_key_string, public_key_string, + std::move(done_cb)); + }, + private_key_strings[i], public_value_strings[i], decryptor_, + record_results[i].cb())); + } + // Verify registration success. + for (auto& record_result : record_results) { + const auto result = record_result.result(); + ASSERT_OK(result.status()) << result.status(); + public_value_ids.push_back(result.ValueOrDie()); + } + + // Encrypt all records in parallel. + std::vector<TestEvent<StatusOr<EncryptedRecord>>> results( + kTestStrings.size()); + for (size_t i = 0; i < kTestStrings.size(); ++i) { + // Choose random key pair. + size_t i_key_pair = base::RandInt(0, public_value_strings.size() - 1); + (new SingleEncryptionContext( + kTestStrings[i], public_value_strings[i_key_pair], + public_value_ids[i_key_pair], encryption_module_, results[i].cb())) + ->Start(); + } + + // Decrypt all records in parallel. + std::vector<TestEvent<StatusOr<std::string>>> decryption_results( + kTestStrings.size()); + for (size_t i = 0; i < results.size(); ++i) { + // Verify encryption success. + const auto result = results[i].result(); + ASSERT_OK(result.status()) << result.status(); + // Decrypt and compare encrypted_record. + (new SingleDecryptionContext( + result.ValueOrDie(), decryptor_, + base::BindOnce( + [](base::OnceCallback<void(StatusOr<std::string>)> + decryption_result, + StatusOr<base::StringPiece> result) { + if (!result.ok()) { + std::move(decryption_result).Run(result.status()); + return; + } + std::move(decryption_result) + .Run(std::string(result.ValueOrDie())); + }, + decryption_results[i].cb()))) + ->Start(); + } + + // Verify decryption results. + for (size_t i = 0; i < decryption_results.size(); ++i) { + const auto decryption_result = decryption_results[i].result(); + ASSERT_OK(decryption_result.status()) << decryption_result.status(); + // Verify data match. + EXPECT_THAT(decryption_result.ValueOrDie(), + ::testing::StrEq(kTestStrings[i])); + } +} +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/encryption/encryption_unittest.cc b/chromium/components/reporting/encryption/encryption_unittest.cc new file mode 100644 index 00000000000..061c11d9aef --- /dev/null +++ b/chromium/components/reporting/encryption/encryption_unittest.cc @@ -0,0 +1,588 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/encryption.h" + +#include "base/bind.h" +#include "base/containers/flat_map.h" +#include "base/hash/hash.h" +#include "base/rand_util.h" +#include "base/strings/strcat.h" +#include "base/synchronization/waitable_event.h" +#include "base/test/task_environment.h" +#include "base/time/time.h" +#include "components/reporting/encryption/decryption.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/status_macros.h" +#include "components/reporting/util/statusor.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" + +namespace reporting { +namespace { + +// Usage (in tests only): +// +// TestEvent<ResType> e; +// ... Do some async work passing e.cb() as a completion callback of +// base::OnceCallback<void(ResType* res)> type which also may perform some +// other action specified by |done| callback provided by the caller. +// ... = e.result(); // Will wait for e.cb() to be called and return the +// collected result. +// +template <typename ResType> +class TestEvent { + public: + TestEvent() : run_loop_(std::make_unique<base::RunLoop>()) {} + ~TestEvent() { EXPECT_FALSE(run_loop_->running()) << "Not responded"; } + TestEvent(const TestEvent& other) = delete; + TestEvent& operator=(const TestEvent& other) = delete; + ResType result() { + run_loop_->Run(); + return std::forward<ResType>(result_); + } + + // Completion callback to hand over to the processing method. + base::OnceCallback<void(ResType res)> cb() { + return base::BindOnce( + [](base::RunLoop* run_loop, ResType* result, ResType res) { + *result = std::forward<ResType>(res); + run_loop->Quit(); + }, + base::Unretained(run_loop_.get()), base::Unretained(&result_)); + } + + private: + std::unique_ptr<base::RunLoop> run_loop_; + ResType result_; +}; + +class EncryptionTest : public ::testing::Test { + protected: + EncryptionTest() = default; + + void SetUp() override { + auto encryptor_result = Encryptor::Create(); + ASSERT_OK(encryptor_result.status()) << encryptor_result.status(); + encryptor_ = std::move(encryptor_result.ValueOrDie()); + + auto decryptor_result = Decryptor::Create(); + ASSERT_OK(decryptor_result.status()) << decryptor_result.status(); + decryptor_ = std::move(decryptor_result.ValueOrDie()); + } + + StatusOr<EncryptedRecord> EncryptSync(base::StringPiece data) { + TestEvent<StatusOr<Encryptor::Handle*>> open_encrypt; + encryptor_->OpenRecord(open_encrypt.cb()); + auto open_encrypt_result = open_encrypt.result(); + RETURN_IF_ERROR(open_encrypt_result.status()); + Encryptor::Handle* const enc_handle = open_encrypt_result.ValueOrDie(); + + TestEvent<Status> add_encrypt; + enc_handle->AddToRecord(data, add_encrypt.cb()); + RETURN_IF_ERROR(add_encrypt.result()); + + EncryptedRecord encrypted; + TestEvent<Status> close_encrypt; + enc_handle->CloseRecord(base::BindOnce( + [](EncryptedRecord* encrypted, + base::OnceCallback<void(Status)> close_cb, + StatusOr<EncryptedRecord> result) { + if (!result.ok()) { + std::move(close_cb).Run(result.status()); + return; + } + *encrypted = result.ValueOrDie(); + std::move(close_cb).Run(Status::StatusOK()); + }, + base::Unretained(&encrypted), close_encrypt.cb())); + RETURN_IF_ERROR(close_encrypt.result()); + return encrypted; + } + + StatusOr<std::string> DecryptSync( + std::pair<std::string /*shared_secret*/, std::string /*encrypted_data*/> + encrypted) { + TestEvent<StatusOr<Decryptor::Handle*>> open_decrypt; + decryptor_->OpenRecord(encrypted.first, open_decrypt.cb()); + auto open_decrypt_result = open_decrypt.result(); + RETURN_IF_ERROR(open_decrypt_result.status()); + Decryptor::Handle* const dec_handle = open_decrypt_result.ValueOrDie(); + + TestEvent<Status> add_decrypt; + dec_handle->AddToRecord(encrypted.second, add_decrypt.cb()); + RETURN_IF_ERROR(add_decrypt.result()); + + std::string decrypted_string; + TestEvent<Status> close_decrypt; + dec_handle->CloseRecord(base::BindOnce( + [](std::string* decrypted_string, + base::OnceCallback<void(Status)> close_cb, + StatusOr<base::StringPiece> result) { + if (!result.ok()) { + std::move(close_cb).Run(result.status()); + return; + } + *decrypted_string = std::string(result.ValueOrDie()); + std::move(close_cb).Run(Status::StatusOK()); + }, + base::Unretained(&decrypted_string), close_decrypt.cb())); + RETURN_IF_ERROR(close_decrypt.result()); + return decrypted_string; + } + + StatusOr<std::string> DecryptMatchingSecret( + Encryptor::PublicKeyId public_key_id, + base::StringPiece encrypted_key) { + // Retrieve private key that matches public key hash. + TestEvent<StatusOr<std::string>> retrieve_private_key; + decryptor_->RetrieveMatchingPrivateKey(public_key_id, + retrieve_private_key.cb()); + ASSIGN_OR_RETURN(std::string private_key, retrieve_private_key.result()); + // Decrypt symmetric key with that private key and peer public key. + ASSIGN_OR_RETURN(std::string shared_secret, + decryptor_->DecryptSecret(private_key, encrypted_key)); + return shared_secret; + } + + Status AddNewKeyPair() { + // Generate new pair of private key and public value. + uint8_t out_public_value[X25519_PUBLIC_VALUE_LEN]; + uint8_t out_private_key[X25519_PRIVATE_KEY_LEN]; + X25519_keypair(out_public_value, out_private_key); + + TestEvent<StatusOr<Encryptor::PublicKeyId>> record_keys; + decryptor_->RecordKeyPair( + std::string(reinterpret_cast<const char*>(out_private_key), + X25519_PRIVATE_KEY_LEN), + std::string(reinterpret_cast<const char*>(out_public_value), + X25519_PUBLIC_VALUE_LEN), + record_keys.cb()); + ASSIGN_OR_RETURN(Encryptor::PublicKeyId new_public_key_id, + record_keys.result()); + TestEvent<Status> set_public_key; + encryptor_->UpdateAsymmetricKey( + std::string(reinterpret_cast<const char*>(out_public_value), + X25519_PUBLIC_VALUE_LEN), + new_public_key_id, set_public_key.cb()); + RETURN_IF_ERROR(set_public_key.result()); + return Status::StatusOK(); + } + + scoped_refptr<Encryptor> encryptor_; + scoped_refptr<Decryptor> decryptor_; + + private: + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; +}; + +TEST_F(EncryptionTest, EncryptAndDecrypt) { + constexpr char kTestString[] = "ABCDEF"; + + // Register new pair of private key and public value. + ASSERT_OK(AddNewKeyPair()); + + // Encrypt the test string using the last public value. + const auto encrypted_result = EncryptSync(kTestString); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + + // Decrypt shared secret with private asymmetric key. + auto decrypt_secret_result = DecryptMatchingSecret( + encrypted_result.ValueOrDie().encryption_info().public_key_id(), + encrypted_result.ValueOrDie().encryption_info().encryption_key()); + ASSERT_OK(decrypt_secret_result.status()) << decrypt_secret_result.status(); + + // Decrypt back. + const auto decrypted_result = DecryptSync( + std::make_pair(decrypt_secret_result.ValueOrDie(), + encrypted_result.ValueOrDie().encrypted_wrapped_record())); + ASSERT_OK(decrypted_result.status()) << decrypted_result.status(); + + EXPECT_THAT(decrypted_result.ValueOrDie(), ::testing::StrEq(kTestString)); +} + +TEST_F(EncryptionTest, NoPublicKey) { + constexpr char kTestString[] = "ABCDEF"; + + // Attempt to encrypt the test string. + const auto encrypted_result = EncryptSync(kTestString); + EXPECT_EQ(encrypted_result.status().error_code(), error::NOT_FOUND); +} + +TEST_F(EncryptionTest, EncryptAndDecryptMultiple) { + constexpr const char* kTestStrings[] = {"Rec1", "Rec22", "Rec333", + "Rec4444", "Rec55555", "Rec666666"}; + // Encrypted records. + std::vector<EncryptedRecord> encrypted_records; + + // 1. Register first key pair. + ASSERT_OK(AddNewKeyPair()); + + // 2. Encrypt 3 test strings. + for (const char* test_string : + {kTestStrings[0], kTestStrings[1], kTestStrings[2]}) { + const auto encrypted_result = EncryptSync(test_string); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + encrypted_records.emplace_back(encrypted_result.ValueOrDie()); + } + + // 3. Register second key pair. + ASSERT_OK(AddNewKeyPair()); + + // 4. Encrypt 2 test strings. + for (const char* test_string : {kTestStrings[3], kTestStrings[4]}) { + const auto encrypted_result = EncryptSync(test_string); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + encrypted_records.emplace_back(encrypted_result.ValueOrDie()); + } + + // 3. Register third key pair. + ASSERT_OK(AddNewKeyPair()); + + // 4. Encrypt one more test strings. + for (const char* test_string : {kTestStrings[5]}) { + const auto encrypted_result = EncryptSync(test_string); + ASSERT_OK(encrypted_result.status()) << encrypted_result.status(); + encrypted_records.emplace_back(encrypted_result.ValueOrDie()); + } + + // For every encrypted record: + for (size_t i = 0; i < encrypted_records.size(); ++i) { + // Decrypt encrypted_key with private asymmetric key. + auto decrypt_secret_result = DecryptMatchingSecret( + encrypted_records[i].encryption_info().public_key_id(), + encrypted_records[i].encryption_info().encryption_key()); + ASSERT_OK(decrypt_secret_result.status()) << decrypt_secret_result.status(); + + // Decrypt back. + const auto decrypted_result = DecryptSync( + std::make_pair(decrypt_secret_result.ValueOrDie(), + encrypted_records[i].encrypted_wrapped_record())); + ASSERT_OK(decrypted_result.status()) << decrypted_result.status(); + + // Verify match. + EXPECT_THAT(decrypted_result.ValueOrDie(), + ::testing::StrEq(kTestStrings[i])); + } +} + +TEST_F(EncryptionTest, EncryptAndDecryptMultipleParallel) { + // Context of single encryption. Self-destructs upon completion or failure. + class SingleEncryptionContext { + public: + SingleEncryptionContext( + base::StringPiece test_string, + base::StringPiece public_key, + Encryptor::PublicKeyId public_key_id, + scoped_refptr<Encryptor> encryptor, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> response) + : test_string_(test_string), + public_key_(public_key), + public_key_id_(public_key_id), + encryptor_(encryptor), + response_(std::move(response)) {} + + SingleEncryptionContext(const SingleEncryptionContext& other) = delete; + SingleEncryptionContext& operator=(const SingleEncryptionContext& other) = + delete; + + ~SingleEncryptionContext() { + DCHECK(!response_) << "Self-destruct without prior response"; + } + + void Start() { + base::ThreadPool::PostTask( + FROM_HERE, base::BindOnce(&SingleEncryptionContext::SetPublicKey, + base::Unretained(this))); + } + + private: + void Respond(StatusOr<EncryptedRecord> result) { + std::move(response_).Run(result); + delete this; + } + void SetPublicKey() { + encryptor_->UpdateAsymmetricKey( + public_key_, public_key_id_, + base::BindOnce( + [](SingleEncryptionContext* self, Status status) { + if (!status.ok()) { + self->Respond(status); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleEncryptionContext::OpenRecord, + base::Unretained(self))); + }, + base::Unretained(this))); + } + void OpenRecord() { + encryptor_->OpenRecord(base::BindOnce( + [](SingleEncryptionContext* self, + StatusOr<Encryptor::Handle*> handle_result) { + if (!handle_result.ok()) { + self->Respond(handle_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleEncryptionContext::AddToRecord, + base::Unretained(self), + base::Unretained(handle_result.ValueOrDie()))); + }, + base::Unretained(this))); + } + void AddToRecord(Encryptor::Handle* handle) { + handle->AddToRecord( + test_string_, + base::BindOnce( + [](SingleEncryptionContext* self, Encryptor::Handle* handle, + Status status) { + if (!status.ok()) { + self->Respond(status); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleEncryptionContext::CloseRecord, + base::Unretained(self), + base::Unretained(handle))); + }, + base::Unretained(this), base::Unretained(handle))); + } + void CloseRecord(Encryptor::Handle* handle) { + handle->CloseRecord(base::BindOnce( + [](SingleEncryptionContext* self, + StatusOr<EncryptedRecord> encryption_result) { + self->Respond(encryption_result); + }, + base::Unretained(this))); + } + + private: + const std::string test_string_; + const std::string public_key_; + const Encryptor::PublicKeyId public_key_id_; + const scoped_refptr<Encryptor> encryptor_; + base::OnceCallback<void(StatusOr<EncryptedRecord>)> response_; + }; + + // Context of single decryption. Self-destructs upon completion or failure. + class SingleDecryptionContext { + public: + SingleDecryptionContext( + const EncryptedRecord& encrypted_record, + scoped_refptr<Decryptor> decryptor, + base::OnceCallback<void(StatusOr<base::StringPiece>)> response) + : encrypted_record_(encrypted_record), + decryptor_(decryptor), + response_(std::move(response)) {} + + SingleDecryptionContext(const SingleDecryptionContext& other) = delete; + SingleDecryptionContext& operator=(const SingleDecryptionContext& other) = + delete; + + ~SingleDecryptionContext() { + DCHECK(!response_) << "Self-destruct without prior response"; + } + + void Start() { + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::RetrieveMatchingPrivateKey, + base::Unretained(this))); + } + + private: + void Respond(StatusOr<base::StringPiece> result) { + std::move(response_).Run(result); + delete this; + } + + void RetrieveMatchingPrivateKey() { + // Retrieve private key that matches public key hash. + decryptor_->RetrieveMatchingPrivateKey( + encrypted_record_.encryption_info().public_key_id(), + base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<std::string> private_key_result) { + if (!private_key_result.ok()) { + self->Respond(private_key_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + &SingleDecryptionContext::DecryptSharedSecret, + base::Unretained(self), + private_key_result.ValueOrDie())); + }, + base::Unretained(this))); + } + + void DecryptSharedSecret(base::StringPiece private_key) { + // Decrypt shared secret from private key and peer public key. + auto shared_secret_result = decryptor_->DecryptSecret( + private_key, encrypted_record_.encryption_info().encryption_key()); + if (!shared_secret_result.ok()) { + Respond(shared_secret_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, base::BindOnce(&SingleDecryptionContext::OpenRecord, + base::Unretained(this), + shared_secret_result.ValueOrDie())); + } + + void OpenRecord(base::StringPiece shared_secret) { + decryptor_->OpenRecord( + shared_secret, + base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<Decryptor::Handle*> handle_result) { + if (!handle_result.ok()) { + self->Respond(handle_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + &SingleDecryptionContext::AddToRecord, + base::Unretained(self), + base::Unretained(handle_result.ValueOrDie()))); + }, + base::Unretained(this))); + } + + void AddToRecord(Decryptor::Handle* handle) { + handle->AddToRecord( + encrypted_record_.encrypted_wrapped_record(), + base::BindOnce( + [](SingleDecryptionContext* self, Decryptor::Handle* handle, + Status status) { + if (!status.ok()) { + self->Respond(status); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::CloseRecord, + base::Unretained(self), + base::Unretained(handle))); + }, + base::Unretained(this), base::Unretained(handle))); + } + + void CloseRecord(Decryptor::Handle* handle) { + handle->CloseRecord(base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<base::StringPiece> decryption_result) { + self->Respond(decryption_result); + }, + base::Unretained(this))); + } + + private: + const EncryptedRecord encrypted_record_; + const scoped_refptr<Decryptor> decryptor_; + base::OnceCallback<void(StatusOr<base::StringPiece>)> response_; + }; + + constexpr std::array<const char*, 6> kTestStrings = { + "Rec1", "Rec22", "Rec333", "Rec4444", "Rec55555", "Rec666666"}; + + // Public and private key pairs in this test are reversed strings. + std::vector<std::string> private_key_strings; + std::vector<std::string> public_value_strings; + std::vector<Encryptor::PublicKeyId> public_value_ids; + for (size_t i = 0; i < 3; ++i) { + // Generate new pair of private key and public value. + uint8_t out_public_value[X25519_PUBLIC_VALUE_LEN]; + uint8_t out_private_key[X25519_PRIVATE_KEY_LEN]; + X25519_keypair(out_public_value, out_private_key); + private_key_strings.emplace_back( + reinterpret_cast<const char*>(out_private_key), X25519_PRIVATE_KEY_LEN); + public_value_strings.emplace_back( + reinterpret_cast<const char*>(out_public_value), + X25519_PUBLIC_VALUE_LEN); + } + + // Register all key pairs for decryption. + std::vector<TestEvent<StatusOr<Encryptor::PublicKeyId>>> record_results( + public_value_strings.size()); + for (size_t i = 0; i < public_value_strings.size(); ++i) { + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce( + [](base::StringPiece private_key_string, + base::StringPiece public_key_string, + scoped_refptr<Decryptor> decryptor, + base::OnceCallback<void(StatusOr<Encryptor::PublicKeyId>)> + done_cb) { + decryptor->RecordKeyPair(private_key_string, public_key_string, + std::move(done_cb)); + }, + private_key_strings[i], public_value_strings[i], decryptor_, + record_results[i].cb())); + } + // Verify registration success. + for (auto& record_result : record_results) { + const auto result = record_result.result(); + ASSERT_OK(result.status()) << result.status(); + public_value_ids.push_back(result.ValueOrDie()); + } + + // Encrypt all records in parallel. + std::vector<TestEvent<StatusOr<EncryptedRecord>>> results( + kTestStrings.size()); + for (size_t i = 0; i < kTestStrings.size(); ++i) { + // Choose random key pair. + size_t i_key_pair = base::RandInt(0, public_value_strings.size() - 1); + (new SingleEncryptionContext( + kTestStrings[i], public_value_strings[i_key_pair], + public_value_ids[i_key_pair], encryptor_, results[i].cb())) + ->Start(); + } + + // Decrypt all records in parallel. + std::vector<TestEvent<StatusOr<std::string>>> decryption_results( + kTestStrings.size()); + for (size_t i = 0; i < results.size(); ++i) { + // Verify encryption success. + const auto result = results[i].result(); + ASSERT_OK(result.status()) << result.status(); + // Decrypt and compare encrypted_record. + (new SingleDecryptionContext( + result.ValueOrDie(), decryptor_, + base::BindOnce( + [](base::OnceCallback<void(StatusOr<std::string>)> + decryption_result, + StatusOr<base::StringPiece> result) { + if (!result.ok()) { + std::move(decryption_result).Run(result.status()); + return; + } + std::move(decryption_result) + .Run(std::string(result.ValueOrDie())); + }, + decryption_results[i].cb()))) + ->Start(); + } + + // Verify decryption results. + for (size_t i = 0; i < decryption_results.size(); ++i) { + const auto decryption_result = decryption_results[i].result(); + ASSERT_OK(decryption_result.status()) << decryption_result.status(); + // Verify data match. + EXPECT_THAT(decryption_result.ValueOrDie(), + ::testing::StrEq(kTestStrings[i])); + } +} + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/encryption/test_encryption_module.cc b/chromium/components/reporting/encryption/test_encryption_module.cc new file mode 100644 index 00000000000..3af2425f6aa --- /dev/null +++ b/chromium/components/reporting/encryption/test_encryption_module.cc @@ -0,0 +1,41 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/test_encryption_module.h" + +#include "base/callback.h" +#include "base/strings/string_piece.h" +#include "components/reporting/encryption/encryption.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/util/statusor.h" + +using ::testing::Invoke; + +namespace reporting { +namespace test { + +TestEncryptionModuleStrict::TestEncryptionModuleStrict() { + ON_CALL(*this, EncryptRecord) + .WillByDefault( + Invoke([](base::StringPiece record, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb) { + EncryptedRecord encrypted_record; + encrypted_record.set_encrypted_wrapped_record(std::string(record)); + // encryption_info is not set. + std::move(cb).Run(encrypted_record); + })); +} + +void TestEncryptionModuleStrict::UpdateAsymmetricKey( + base::StringPiece new_public_key, + Encryptor::PublicKeyId new_public_key_id, + base::OnceCallback<void(Status)> response_cb) { + // Ignore keys but return success. + std::move(response_cb).Run(Status(Status::StatusOK())); +} + +TestEncryptionModuleStrict::~TestEncryptionModuleStrict() = default; + +} // namespace test +} // namespace reporting diff --git a/chromium/components/reporting/encryption/test_encryption_module.h b/chromium/components/reporting/encryption/test_encryption_module.h new file mode 100644 index 00000000000..498d15a5095 --- /dev/null +++ b/chromium/components/reporting/encryption/test_encryption_module.h @@ -0,0 +1,46 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_ENCRYPTION_TEST_ENCRYPTION_MODULE_H_ +#define COMPONENTS_REPORTING_ENCRYPTION_TEST_ENCRYPTION_MODULE_H_ + +#include "base/callback.h" +#include "base/strings/string_piece.h" +#include "components/reporting/encryption/encryption.h" +#include "components/reporting/encryption/encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/util/statusor.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace reporting { +namespace test { + +// An |EncryptionModule| that does no encryption. +class TestEncryptionModuleStrict : public EncryptionModule { + public: + TestEncryptionModuleStrict(); + + MOCK_METHOD(void, + EncryptRecord, + (base::StringPiece record, + base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb), + (const override)); + + void UpdateAsymmetricKey( + base::StringPiece new_public_key, + Encryptor::PublicKeyId new_public_key_id, + base::OnceCallback<void(Status)> response_cb) override; + + protected: + ~TestEncryptionModuleStrict() override; +}; + +// Most of the time no need to log uninterested calls to |EncryptRecord|. +typedef ::testing::NiceMock<TestEncryptionModuleStrict> TestEncryptionModule; + +} // namespace test +} // namespace reporting + +#endif // COMPONENTS_REPORTING_ENCRYPTION_TEST_ENCRYPTION_MODULE_H_ diff --git a/chromium/components/reporting/encryption/verification.cc b/chromium/components/reporting/encryption/verification.cc new file mode 100644 index 00000000000..5e5bcfdb4bf --- /dev/null +++ b/chromium/components/reporting/encryption/verification.cc @@ -0,0 +1,60 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/verification.h" + +#include "components/reporting/util/status.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" + +namespace reporting { + +namespace { + +// Well-known public signature verification keys. +constexpr uint8_t kProdVerificationKey[ED25519_PUBLIC_KEY_LEN] = { + 0x51, 0x2D, 0x53, 0xA3, 0xF5, 0xEB, 0x01, 0xCE, 0xDA, 0xFC, 0x8E, + 0x79, 0xE7, 0x0F, 0xE1, 0x65, 0xDC, 0x14, 0x86, 0x53, 0x8B, 0x97, + 0x5A, 0x2D, 0x70, 0x08, 0xCB, 0xCA, 0x60, 0xC3, 0x55, 0xE6}; + +constexpr uint8_t kDevVerificationKey[ED25519_PUBLIC_KEY_LEN] = { + 0xC6, 0x2C, 0x4D, 0x25, 0x9E, 0x3E, 0x99, 0xA0, 0x2E, 0x08, 0x15, + 0x8C, 0x38, 0xB7, 0x6C, 0x08, 0xDF, 0xE7, 0x6E, 0x3A, 0xD6, 0x5A, + 0xC5, 0x58, 0x09, 0xE4, 0xAB, 0x89, 0x3A, 0x31, 0x53, 0x07}; + +} // namespace + +// static +base::StringPiece SignatureVerifier::VerificationKey() { + return base::StringPiece(reinterpret_cast<const char*>(kProdVerificationKey), + ED25519_PUBLIC_KEY_LEN); +} + +// static +base::StringPiece SignatureVerifier::VerificationKeyDev() { + return base::StringPiece(reinterpret_cast<const char*>(kDevVerificationKey), + ED25519_PUBLIC_KEY_LEN); +} + +SignatureVerifier::SignatureVerifier(base::StringPiece verification_public_key) + : verification_public_key_(verification_public_key) {} + +Status SignatureVerifier::Verify(base::StringPiece message, + base::StringPiece signature) { + if (signature.size() != ED25519_SIGNATURE_LEN) { + return Status{error::FAILED_PRECONDITION, "Wrong signature size"}; + } + if (verification_public_key_.size() != ED25519_PUBLIC_KEY_LEN) { + return Status{error::FAILED_PRECONDITION, "Wrong public key size"}; + } + const int result = ED25519_verify( + reinterpret_cast<const uint8_t*>(message.data()), message.size(), + reinterpret_cast<const uint8_t*>(signature.data()), + reinterpret_cast<const uint8_t*>(verification_public_key_.data())); + if (result != 1) { + return Status{error::INVALID_ARGUMENT, "Verification failed"}; + } + return Status::StatusOK(); +} + +} // namespace reporting diff --git a/chromium/components/reporting/encryption/verification.h b/chromium/components/reporting/encryption/verification.h new file mode 100644 index 00000000000..0b3d3d1b9ae --- /dev/null +++ b/chromium/components/reporting/encryption/verification.h @@ -0,0 +1,38 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_ENCRYPTION_VERIFICATION_H_ +#define COMPONENTS_REPORTING_ENCRYPTION_VERIFICATION_H_ + +#include <string> + +#include "base/strings/string_piece.h" +#include "components/reporting/util/status.h" + +namespace reporting { + +// Helper class that verifies an Ed25519 signed message received from +// the server. It uses boringssl implementation available on the client. +class SignatureVerifier { + public: + // Well-known public signature verification keys that is used to verify + // that signed data is indeed originating from reporting server. + // Exists in two flavors: PROD and DEV. + static base::StringPiece VerificationKey(); + static base::StringPiece VerificationKeyDev(); + + // Ed25519 |verification_public_key| must consist of ED25519_PUBLIC_KEY_LEN + // bytes. + explicit SignatureVerifier(base::StringPiece verification_public_key); + + // Actual verification - returns error status if provided |signature| does not + // match |message|. Signature must be ED25519_SIGNATURE_LEN bytes. + Status Verify(base::StringPiece message, base::StringPiece signature); + + private: + std::string verification_public_key_; +}; +} // namespace reporting + +#endif // COMPONENTS_REPORTING_ENCRYPTION_VERIFICATION_H_ diff --git a/chromium/components/reporting/encryption/verification_unittest.cc b/chromium/components/reporting/encryption/verification_unittest.cc new file mode 100644 index 00000000000..09e7df6592e --- /dev/null +++ b/chromium/components/reporting/encryption/verification_unittest.cc @@ -0,0 +1,151 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/encryption/verification.h" + +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" + +using ::testing::Eq; +using ::testing::HasSubstr; + +namespace reporting { +namespace { + +class VerificationTest : public ::testing::Test { + protected: + VerificationTest() = default; + void SetUp() override { + // Generate key pair + ED25519_keypair(public_key_, private_key_); + } + + uint8_t public_key_[ED25519_PUBLIC_KEY_LEN]; + uint8_t private_key_[ED25519_PRIVATE_KEY_LEN]; +}; + +TEST_F(VerificationTest, SignAndVerify) { + static constexpr char message[] = "ABCDEF 012345"; + // Sign a message. + uint8_t signature[ED25519_SIGNATURE_LEN]; + ASSERT_THAT(ED25519_sign(signature, reinterpret_cast<const uint8_t*>(message), + strlen(message), private_key_), + Eq(1)); + + // Verify the signature. + SignatureVerifier verifier(std::string( + reinterpret_cast<const char*>(public_key_), ED25519_PUBLIC_KEY_LEN)); + EXPECT_OK( + verifier.Verify(std::string(message, strlen(message)), + std::string(reinterpret_cast<const char*>(signature), + ED25519_SIGNATURE_LEN))); +} + +TEST_F(VerificationTest, SignAndFailBadSignature) { + static constexpr char message[] = "ABCDEF 012345"; + // Sign a message. + uint8_t signature[ED25519_SIGNATURE_LEN]; + ASSERT_THAT(ED25519_sign(signature, reinterpret_cast<const uint8_t*>(message), + strlen(message), private_key_), + Eq(1)); + + // Verify the signature - wrong length. + SignatureVerifier verifier(std::string( + reinterpret_cast<const char*>(public_key_), ED25519_PUBLIC_KEY_LEN)); + Status status = + verifier.Verify(std::string(message, strlen(message)), + std::string(reinterpret_cast<const char*>(signature), + ED25519_SIGNATURE_LEN - 1)); + EXPECT_THAT(status.code(), Eq(error::FAILED_PRECONDITION)); + EXPECT_THAT(status.message(), HasSubstr("Wrong signature size")); + + // Verify the signature - mismatch. + signature[0] = ~signature[0]; + status = verifier.Verify(std::string(message, strlen(message)), + std::string(reinterpret_cast<const char*>(signature), + ED25519_SIGNATURE_LEN)); + EXPECT_THAT(status.code(), Eq(error::INVALID_ARGUMENT)); + EXPECT_THAT(status.message(), HasSubstr("Verification failed")); +} + +TEST_F(VerificationTest, SignAndFailBadPublicKey) { + static constexpr char message[] = "ABCDEF 012345"; + // Sign a message. + uint8_t signature[ED25519_SIGNATURE_LEN]; + ASSERT_THAT(ED25519_sign(signature, reinterpret_cast<const uint8_t*>(message), + strlen(message), private_key_), + Eq(1)); + + // Verify the public key - wrong length. + SignatureVerifier verifier(std::string( + reinterpret_cast<const char*>(public_key_), ED25519_PUBLIC_KEY_LEN - 1)); + Status status = + verifier.Verify(std::string(message, strlen(message)), + std::string(reinterpret_cast<const char*>(signature), + ED25519_SIGNATURE_LEN)); + EXPECT_THAT(status.code(), Eq(error::FAILED_PRECONDITION)); + EXPECT_THAT(status.message(), HasSubstr("Wrong public key size")); + + // Verify the public key - mismatch. + public_key_[0] = ~public_key_[0]; + SignatureVerifier verifier2(std::string( + reinterpret_cast<const char*>(public_key_), ED25519_PUBLIC_KEY_LEN)); + status = + verifier2.Verify(std::string(message, strlen(message)), + std::string(reinterpret_cast<const char*>(signature), + ED25519_SIGNATURE_LEN)); + EXPECT_THAT(status.code(), Eq(error::INVALID_ARGUMENT)); + EXPECT_THAT(status.message(), HasSubstr("Verification failed")); +} + +TEST_F(VerificationTest, ValidateFixedKey) { + // |dev_data_to_sign| is signed on DEV server, producing + // |dev_server_signature|. + static constexpr uint8_t dev_data_to_sign[] = { + 0x4D, 0x22, 0x5C, 0x4C, 0x74, 0x23, 0x82, 0x80, 0x58, 0xA2, 0x31, 0xA2, + 0xC6, 0xE2, 0x6D, 0xDA, 0x48, 0x82, 0x7A, 0x9C, 0xF7, 0xD0, 0x4A, 0xF2, + 0xFD, 0x19, 0x03, 0x7F, 0xC5, 0x6F, 0xBB, 0x49, 0xAF, 0x91, 0x7B, 0x74}; + static constexpr uint8_t dev_server_signature[ED25519_SIGNATURE_LEN] = { + 0x0C, 0xA4, 0xAF, 0xE3, 0x27, 0x06, 0xD1, 0x4F, 0x0E, 0x05, 0x44, + 0x74, 0x0D, 0x4F, 0xA0, 0x4C, 0x26, 0xB1, 0x0C, 0x44, 0x92, 0x0F, + 0x96, 0xAF, 0x5A, 0x7E, 0x45, 0xED, 0x61, 0xB7, 0x87, 0xA8, 0xA3, + 0x98, 0x52, 0x97, 0x8D, 0x56, 0xA3, 0xED, 0xF7, 0x9B, 0x54, 0x17, + 0x61, 0x32, 0x6C, 0x06, 0x29, 0xBF, 0x30, 0x4E, 0x88, 0x72, 0xAB, + 0xE3, 0x60, 0xDA, 0xF0, 0x37, 0xEB, 0x56, 0x28, 0x0B}; + + // Validate the signature using known DEV public key. + SignatureVerifier dev_verifier(SignatureVerifier::VerificationKeyDev()); + const auto dev_result = dev_verifier.Verify( + std::string(reinterpret_cast<const char*>(dev_data_to_sign), + sizeof(dev_data_to_sign)), + std::string(reinterpret_cast<const char*>(dev_server_signature), + ED25519_SIGNATURE_LEN)); + EXPECT_OK(dev_result) << dev_result; + + // |prod_data_to_sign| is signed on PROD server, producing + // |prod_server_signature|. + static constexpr uint8_t prod_data_to_sign[] = { + 0xB3, 0xF9, 0xA3, 0xCC, 0xEB, 0x42, 0x88, 0x6b, 0x3f, 0x7b, 0x93, 0xC3, + 0xD3, 0x61, 0x9C, 0x45, 0xB4, 0xD7, 0x4B, 0x7B, 0x4F, 0xA7, 0x1A, 0x29, + 0xE1, 0x95, 0x14, 0xA4, 0x8C, 0x21, 0x36, 0x9F, 0x34, 0xA7, 0x4A, 0x57}; + static constexpr uint8_t prod_server_signature[ED25519_SIGNATURE_LEN] = { + 0x17, 0xA4, 0x18, 0xA3, 0x78, 0x7A, 0x75, 0x24, 0xD9, 0x81, 0x3D, + 0x9F, 0x17, 0x28, 0x40, 0xD8, 0xE7, 0x67, 0x88, 0x17, 0x44, 0x7C, + 0xC2, 0x1A, 0xE2, 0x73, 0xAC, 0xB1, 0x0B, 0xCE, 0x60, 0xBB, 0x30, + 0x58, 0xCE, 0xF6, 0x8E, 0x33, 0xB6, 0xC6, 0x18, 0x3C, 0xA7, 0xD4, + 0x38, 0x91, 0x90, 0x2C, 0xBC, 0xB9, 0x76, 0x3C, 0xFF, 0x6F, 0x84, + 0xEC, 0x2D, 0x1E, 0x73, 0x43, 0x1B, 0x75, 0x5E, 0x0E}; + + // Validate the signature using known PROD public key. + SignatureVerifier prod_verifier(SignatureVerifier::VerificationKey()); + const auto prod_result = prod_verifier.Verify( + std::string(reinterpret_cast<const char*>(prod_data_to_sign), + sizeof(prod_data_to_sign)), + std::string(reinterpret_cast<const char*>(prod_server_signature), + ED25519_SIGNATURE_LEN)); + EXPECT_OK(prod_result) << prod_result; +} +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/proto/BUILD.gn b/chromium/components/reporting/proto/BUILD.gn new file mode 100644 index 00000000000..9eb8f80a2cc --- /dev/null +++ b/chromium/components/reporting/proto/BUILD.gn @@ -0,0 +1,22 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. Use +# of this source code is governed by a BSD-style license that can +# be found in the LICENSE file. + +import("//third_party/libprotobuf-mutator/fuzzable_proto_library.gni") +import("//third_party/protobuf/proto_library.gni") + +# Record constants for use with the reporting messaging library. +proto_library("record_constants") { + sources = [ "record_constants.proto" ] + + proto_out_dir = "components/reporting/proto" +} + +# Record definitions for reporting. +proto_library("record_proto") { + sources = [ "record.proto" ] + + deps = [ ":record_constants" ] + + proto_out_dir = "components/reporting/proto" +} diff --git a/chromium/components/reporting/proto/record.proto b/chromium/components/reporting/proto/record.proto new file mode 100644 index 00000000000..9c03bd430af --- /dev/null +++ b/chromium/components/reporting/proto/record.proto @@ -0,0 +1,125 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package reporting; + +import "record_constants.proto"; + +// Record represents the data sent from the Reporting Client. +message Record { + // Record data as enqueued with an ReportingQueue::Enqueue call (required). + // Data structure requirements are set by the destination. For destinations + // expecting a proto - the proto will be MessageLite::SerializeToString(), and + // will be DeserializedFromString() in the destination handler, prior to being + // forwarded. + // + // Current expected formats (destination : type): + // Destination::UPLOAD_EVENTS : UploadEventsRequest + optional bytes data = 1; + + // The destination associated with this request as set with the + // ReportingQueueConfiguration (required). + optional Destination destination = 2; + + // The DMToken associated with this request as set with the + // ReportingQueueConfiuguration (required). + optional string dm_token = 3; + + // When the record was submitted to ReportingQueue::Enqueue. + // Represents UTC time since Unix epoch 1970-01-01T00:00:00Z in microseconds, + // to match Spanner timestamp format. + optional int64 timestamp_us = 4; +} + +// A Record with it's digest and the digest of the previous record. +message WrappedRecord { + // Record (required) + // Data provided by the Reporting Client. + optional Record record = 1; + + // Record Digest (required) + // SHA256 hash used to validate that the record has been retrieved without + // being manipulated while it was on the device or during transfer. + optional bytes record_digest = 2; + + // Last record digest (required) + // Created by client and used by server to verify that the sequence of records + // has not been tampered with. + optional bytes last_record_digest = 3; +} + +// Information about how the record was encrypted. +message EncryptionInfo { + // Encryption key (optional). + // Represents a symmetric key used for |encrypted_wrapped_record| + // encryption; itself encrypted with asymmetric encryption by a public key. + // The private portion of the key is known to the receiver only, and + // identified with the |public_key_id|. + optional bytes encryption_key = 1; + + // Public key id (optional) + // Hash of the public key used to do encryption. Used to identity the + // private key for decryption. If no key_id is present, it is assumed that + // |key| has been transferred in plaintext. + optional int64 public_key_id = 2; +} + +// Tracking information for what order a record appears in. +message SequencingInformation { + // Sequencing ID (monotonic number, required) + // Tracks records processing progress and is used for confirming that this + // and all prior records have been processed. If the same number is + // encountered more than once, only one instance needs to be processed. If + // certain number is absent when higher are encountered, it indicates that + // some records have been lost and there is a gap in the records stream + // (what to do with that is a decision that the caller needs to make). + optional int64 sequencing_id = 1; + + // Generation ID (required). Unique per device and priority. Generated anew + // when previous record digest is not found at startup (e.g. after powerwash). + optional int64 generation_id = 2; + + // Priority (required). + optional Priority priority = 3; +} + +// |WrappedRecord| as it is stored on disc, and sent to the server. +message EncryptedRecord { + // Encrypted Wrapped Record + // |WrappedRecord| encrypted with the |encryption_key| in |encryption_info|. + // When absent, indicates gap - respective record is irreparably corrupt or + // missing from Storage, and server side should log it as such and no longer + // expect client to deliver it. + optional bytes encrypted_wrapped_record = 1; + + // Must be present to facilitate decryption of encrypted record. + // If missing, the record is either not encrypted or missing. + // TODO(b/153651358): Disable an option to send record not encrypted. + optional EncryptionInfo encryption_info = 2; + + // Sequencing information (required). Must be present to allow + // tracking and confirmation of the events by server. + optional SequencingInformation sequencing_information = 3; +} + +// Encryption public key as delivered from the server and stored in Storage. +// Signature ensures the key was actually sent by the server and not manipulated +// afterwards. +message SignedEncryptionInfo { + // Public asymmetric key (required). + optional bytes public_asymmetric_key = 1; + + // Public key id (required). + // Identifies private key matching |public_asymmetric_key| for the server. + // Matches Encryptor::PublicKeyId. + optional int32 public_key_id = 2; + + // Signature of |public_asymmetric_key| (required). + // Verified by client against a well-known signature. + optional bytes signature = 3; +} diff --git a/chromium/components/reporting/proto/record_constants.proto b/chromium/components/reporting/proto/record_constants.proto new file mode 100644 index 00000000000..5a8d383ac95 --- /dev/null +++ b/chromium/components/reporting/proto/record_constants.proto @@ -0,0 +1,88 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package reporting; + +// |Destination| indicates which handler a |Record| should be delivered to. +enum Destination { + UNDEFINED_DESTINATION = 0; + + // |UPLOAD_EVENTS| handler sends records to the Eventing pipeline. + UPLOAD_EVENTS = 1; + + // |MEET_DEVICE_TELEMETRY| handler is for telemetry data sent by Meet + // Devices. For more information, see go/reliable-meet-device-telemetry. + MEET_DEVICE_TELEMETRY = 2; + + // |WEB_PROTECT| legacy web protect event handler + WEB_PROTECT = 3; + + // |ARC_INSTALL| legacy arc app installation event handler + ARC_INSTALL = 4; + + // |POLICY_VALIDATION| legacy policy validation event handler + POLICY_VALIDATION = 5; + + // |EXTENSION_INSTALL| legacy extension installation event handler + EXTENSION_INSTALL = 6; + + // |REPORTING_RECORD| Temporary group for Encrypted Reporting Pipeline + REPORTING_RECORD = 7; + + // |DEVICE_TRUST_REPORTS| handler is for reporting data updates related to the + // Device Trust connector, sent by Chrome browsers for Chrome Browser Cloud + // Management (CBCM). + DEVICE_TRUST_REPORTS = 8; +} + +// |Priority| is used to determine when items from the queue should be rate +// limited or shed. Rate limiting indicates that fewer records will be sent due +// to message volume, records of the lowest priority are limited first. Shedding +// records occurs when disk space is at or near the limt, records of the lowest +// priority are shed first. +enum Priority { + UNDEFINED_PRIORITY = 0; + + // |IMMEDIATE| queues should transfer small amounts of immediately necessary + // information. These are the events that will be rate limited last. + // |IMMEDIATE| records are the last ones to be shed. + // Security events are the only example of events that need to be |IMMEDIATE|. + IMMEDIATE = 1; + + // |FAST_BATCH| queues should transfer small amounts of information that may + // be critical for administrative experience. These records will be rate + // limited before |IMMEDIATE| records. + // |FAST_BATCH| records are shed before |IMMEDIATE| records. + // Resource utilization and failed application installation are perfect + // examples of records that need to be |FAST_BATCH|. + FAST_BATCH = 2; + + // |SLOW_BATCH| queues should transfer small amounts of non-immediate data. + // These records will be rate limited before |FAST_BATCH| records. + // |SLOW_BATCH| records are shed before |FAST_BATCH| records. + // Application metrics are a good example of records that should be + // |SLOW_BATCH|. + SLOW_BATCH = 3; + + // |BACKGROUND_BATCH| queues transfer large amounts of non-immediate data. + // These records will be rate limited before |SLOW_BATCH| records. + // |BACKGROUND_BATCH| records are shed before |SLOW_BATCH| records. + // Log files are a perfect examples of records that need to be + // |BACKGROUND_BATCH|. + BACKGROUND_BATCH = 4; + + // |MANUAL_BATCH| queues transfer data only on explicit request. + // Note that since a queue can hold records submitted by multiple clients, + // one client requesting to transfer data will do so for all collected + // records of the same priority, including those enqueued by other clients. + // |MANUAL_BATCH| records are the first to be rate limited, and since there + // is no automatic transfer, it is important to explicitly flush them often + // enough to avoid loss of data. + // |MANUAL_BATCH| records are the first to be shed. + MANUAL_BATCH = 5; +} diff --git a/chromium/components/reporting/storage/BUILD.gn b/chromium/components/reporting/storage/BUILD.gn new file mode 100644 index 00000000000..34001b3098b --- /dev/null +++ b/chromium/components/reporting/storage/BUILD.gn @@ -0,0 +1,173 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD - style license that can be +# found in the LICENSE file. + +import("//build/config/features.gni") + +static_library("storage_configuration") { + sources = [ + "storage_configuration.cc", + "storage_configuration.h", + ] + + deps = [ "//base" ] +} + +static_library("storage_uploader_interface") { + sources = [ + "storage_uploader_interface.cc", + "storage_uploader_interface.h", + ] + + deps = [ + "//base", + "//components/reporting/proto:record_constants", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + "//components/reporting/util:status_proto", + ] +} + +static_library("storage_queue") { + sources = [ + "storage_queue.cc", + "storage_queue.h", + ] + + deps = [ + ":storage_configuration", + ":storage_uploader_interface", + "//base", + "//components/reporting/encryption:encryption_module", + "//components/reporting/encryption:verification", + "//components/reporting/proto:record_constants", + "//components/reporting/proto:record_proto", + "//components/reporting/storage/resources:resource_interface", + "//components/reporting/util:status", + "//components/reporting/util:status_macros", + "//components/reporting/util:task_runner_context", + "//crypto", + "//third_party/protobuf:protobuf_lite", + ] +} + +static_library("storage") { + sources = [ + "storage.cc", + "storage.h", + ] + + public_deps = [ ":storage_configuration" ] + + deps = [ + ":storage_queue", + ":storage_uploader_interface", + "//base", + "//components/reporting/encryption:encryption_module", + "//components/reporting/encryption:verification", + "//components/reporting/proto:record_constants", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + "//components/reporting/util:status_macros", + "//components/reporting/util:task_runner_context", + "//third_party/boringssl", + "//third_party/protobuf:protobuf_lite", + ] +} + +static_library("storage_module") { + sources = [ + "storage_module.cc", + "storage_module.h", + "storage_module_interface.cc", + "storage_module_interface.h", + ] + + public_deps = [ ":storage_configuration" ] + + deps = [ + ":storage", + ":storage_uploader_interface", + "//base", + "//components/reporting/encryption:encryption_module", + "//components/reporting/proto:record_constants", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + ] +} + +static_library("missive_storage_module") { + sources = [ + "missive_storage_module.cc", + "missive_storage_module.h", + "storage_module_interface.cc", + "storage_module_interface.h", + ] + + public_deps = [ ":storage_configuration" ] + + deps = [ + ":storage", + ":storage_uploader_interface", + "//base", + "//components/reporting/proto:record_constants", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + ] +} + +static_library("test_support") { + testonly = true + sources = [ + "test_storage_module.cc", + "test_storage_module.h", + ] + public_deps = [ + ":storage", + ":storage_configuration", + ":storage_module", + ":storage_queue", + "//components/reporting/proto:record_constants", + "//components/reporting/proto:record_proto", + "//components/reporting/util:status", + ] + deps = [ + "//base", + "//crypto", + "//testing/gmock", + "//testing/gtest", + "//third_party/boringssl", + ] +} + +# All unit tests are built as part of the //components:components_unittests +# target and must be one targets named "unit_tests". +source_set("unit_tests") { + testonly = true + sources = [ + "storage_queue_stress_test.cc", + "storage_queue_unittest.cc", + "storage_unittest.cc", + ] + deps = [ + ":storage", + ":storage_configuration", + ":storage_module", + ":storage_queue", + ":storage_uploader_interface", + ":test_support", + "//base", + "//base/test:test_support", + "//components/reporting/encryption:decryption", + "//components/reporting/encryption:encryption", + "//components/reporting/encryption:test_support", + "//components/reporting/proto:record_proto", + "//components/reporting/storage/resources:resource_interface", + "//components/reporting/util:status", + "//components/reporting/util:status_macros", + "//crypto", + "//testing/gmock", + "//testing/gtest", + "//third_party/boringssl", + ] +} diff --git a/chromium/components/reporting/storage/DEPS b/chromium/components/reporting/storage/DEPS new file mode 100644 index 00000000000..75318e72580 --- /dev/null +++ b/chromium/components/reporting/storage/DEPS @@ -0,0 +1,7 @@ +include_rules = [ + "+base", + "+crypto", + "+third_party/protobuf", + "+third_party/boringssl/src/include", +] + diff --git a/chromium/components/reporting/storage/missive_storage_module.cc b/chromium/components/reporting/storage/missive_storage_module.cc new file mode 100644 index 00000000000..4693616492d --- /dev/null +++ b/chromium/components/reporting/storage/missive_storage_module.cc @@ -0,0 +1,57 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/missive_storage_module.h" + +#include <memory> +#include <utility> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/memory/ptr_util.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/util/status.h" + +namespace reporting { + +using MissiveStorageModuleDelegateInterface = + MissiveStorageModule::MissiveStorageModuleDelegateInterface; + +MissiveStorageModuleDelegateInterface::MissiveStorageModuleDelegateInterface() = + default; +MissiveStorageModuleDelegateInterface:: + ~MissiveStorageModuleDelegateInterface() = default; + +MissiveStorageModule::MissiveStorageModule( + std::unique_ptr<MissiveStorageModuleDelegateInterface> delegate) + : delegate_(std::move(delegate)) {} + +MissiveStorageModule::~MissiveStorageModule() = default; + +// static +scoped_refptr<MissiveStorageModule> MissiveStorageModule::Create( + std::unique_ptr<MissiveStorageModuleDelegateInterface> delegate) { + return base::WrapRefCounted(new MissiveStorageModule(std::move(delegate))); +} + +void MissiveStorageModule::AddRecord( + Priority priority, + Record record, + base::OnceCallback<void(Status)> callback) { + delegate_->AddRecord(priority, record, std::move(callback)); +} + +void MissiveStorageModule::ReportSuccess( + SequencingInformation sequencing_information, + bool force) { + delegate_->ReportSuccess(sequencing_information, force); +} + +void MissiveStorageModule::UpdateEncryptionKey( + SignedEncryptionInfo signed_encryption_info) { + delegate_->UpdateEncryptionKey(signed_encryption_info); +} + +} // namespace reporting diff --git a/chromium/components/reporting/storage/missive_storage_module.h b/chromium/components/reporting/storage/missive_storage_module.h new file mode 100644 index 00000000000..bdb0e35327e --- /dev/null +++ b/chromium/components/reporting/storage/missive_storage_module.h @@ -0,0 +1,89 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_MISSIVE_STORAGE_MODULE_H_ +#define COMPONENTS_REPORTING_STORAGE_MISSIVE_STORAGE_MODULE_H_ + +#include <utility> + +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/storage/storage_module_interface.h" +#include "components/reporting/util/status.h" + +namespace reporting { + +// MissiveStorageModule is initialized by the |MissiveClient|, in order to get a +// copy call |MissiveClient::GetMissiveStorageModule|. +// +// MissiveStorageModule utilizes a Delegate and forwards all calls to the +// delegate. +class MissiveStorageModule : public StorageModuleInterface { + public: + // MissiveStorageModuleDelegateInterface has the same interface as + // StorageModuleInterface but isn't shared or created as a scoped_refptr. + // MissiveStorageModuleDelegateInterface is expected to be implemented by the + // caller. + class MissiveStorageModuleDelegateInterface { + public: + MissiveStorageModuleDelegateInterface(); + virtual ~MissiveStorageModuleDelegateInterface(); + MissiveStorageModuleDelegateInterface( + const MissiveStorageModuleDelegateInterface& other) = delete; + MissiveStorageModuleDelegateInterface& operator=( + const MissiveStorageModuleDelegateInterface& other) = delete; + + virtual void AddRecord(Priority priority, + Record record, + base::OnceCallback<void(Status)> callback) = 0; + virtual void ReportSuccess(SequencingInformation sequencing_information, + bool force) = 0; + virtual void UpdateEncryptionKey( + SignedEncryptionInfo signed_encryption_key) = 0; + }; + + // Factory method creates |MissiveStorageModule| object. + static scoped_refptr<MissiveStorageModule> Create( + std::unique_ptr<MissiveStorageModuleDelegateInterface> delegate); + + MissiveStorageModule(const MissiveStorageModule& other) = delete; + MissiveStorageModule& operator=(const MissiveStorageModule& other) = delete; + + // Calls |missive_delegate_->AddRecord| forwarding the arguments. + void AddRecord(Priority priority, + Record record, + base::OnceCallback<void(Status)> callback) override; + + // Once a record has been successfully uploaded, the sequencing information + // can be passed back to the StorageModule here for record deletion. + // If |force| is false (which is used in most cases), |sequencing_information| + // only affects Storage if no higher sequeincing was confirmed before; + // otherwise it is accepted unconditionally. + void ReportSuccess(SequencingInformation sequencing_information, + bool force) override; + + // If the server attached signed encryption key to the response, it needs to + // be paased here. + void UpdateEncryptionKey(SignedEncryptionInfo signed_encryption_key) override; + + protected: + // Constructor can only be called by |Create| factory method. + explicit MissiveStorageModule( + std::unique_ptr<MissiveStorageModuleDelegateInterface> delegate); + + // Refcounted object must have destructor declared protected or private. + ~MissiveStorageModule() override; + + private: + friend base::RefCountedThreadSafe<MissiveStorageModule>; + + std::unique_ptr<MissiveStorageModuleDelegateInterface> delegate_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_MISSIVE_STORAGE_MODULE_H_ diff --git a/chromium/components/reporting/storage/resources/BUILD.gn b/chromium/components/reporting/storage/resources/BUILD.gn new file mode 100644 index 00000000000..69410bec66a --- /dev/null +++ b/chromium/components/reporting/storage/resources/BUILD.gn @@ -0,0 +1,34 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD - style license that can be +# found in the LICENSE file. + +import("//build/config/features.gni") + +static_library("resource_interface") { + visibility = [ "//components/reporting/storage/*" ] + sources = [ + "disk_resource_impl.cc", + "disk_resource_impl.h", + "memory_resource_impl.cc", + "memory_resource_impl.h", + "resource_interface.cc", + "resource_interface.h", + ] + + deps = [ "//base" ] +} + +# All unit tests are built as part of the //components:components_unittests +# target and must be one targets named "unit_tests". +# TODO(chromium:1169835) These tests can't be run on iOS until they are updated. +source_set("unit_tests") { + testonly = true + sources = [ "resource_interface_unittest.cc" ] + deps = [ + ":resource_interface", + "//base", + "//base/test:test_support", + "//testing/gmock", + "//testing/gtest", + ] +} diff --git a/chromium/components/reporting/storage/resources/disk_resource_impl.cc b/chromium/components/reporting/storage/resources/disk_resource_impl.cc new file mode 100644 index 00000000000..c8ca7b4bf2c --- /dev/null +++ b/chromium/components/reporting/storage/resources/disk_resource_impl.cc @@ -0,0 +1,54 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/resources/disk_resource_impl.h" + +#include <atomic> +#include <cstdint> + +#include "base/check_op.h" +#include "base/no_destructor.h" + +namespace reporting { + +// TODO(b/159361496): Set total disk allowance based on the platform +// (or policy?). +DiskResourceImpl::DiskResourceImpl() + : total_(256u * 1024LLu * 1024LLu), // 256 MiB + used_(0) {} + +DiskResourceImpl::~DiskResourceImpl() = default; + +bool DiskResourceImpl::Reserve(uint64_t size) { + uint64_t old_used = used_.fetch_add(size); + if (old_used + size > total_) { + used_.fetch_sub(size); + return false; + } + return true; +} + +void DiskResourceImpl::Discard(uint64_t size) { + DCHECK_LE(size, used_.load()); + used_.fetch_sub(size); +} + +uint64_t DiskResourceImpl::GetTotal() { + return total_; +} + +uint64_t DiskResourceImpl::GetUsed() { + return used_.load(); +} + +void DiskResourceImpl::Test_SetTotal(uint64_t test_total) { + total_ = test_total; +} + +ResourceInterface* GetDiskResource() { + static base::NoDestructor<DiskResourceImpl> disk; + return disk.get(); +} + +} // namespace reporting diff --git a/chromium/components/reporting/storage/resources/disk_resource_impl.h b/chromium/components/reporting/storage/resources/disk_resource_impl.h new file mode 100644 index 00000000000..927fff1086c --- /dev/null +++ b/chromium/components/reporting/storage/resources/disk_resource_impl.h @@ -0,0 +1,37 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_RESOURCES_DISK_RESOURCE_IMPL_H_ +#define COMPONENTS_REPORTING_STORAGE_RESOURCES_DISK_RESOURCE_IMPL_H_ + +#include <atomic> +#include <cstdint> + +#include "components/reporting/storage/resources/resource_interface.h" + +namespace reporting { + +// Interface to resources management by Storage module. +// Must be implemented by the caller base on the platform limitations. +// All APIs are non-blocking. +class DiskResourceImpl : public ResourceInterface { + public: + DiskResourceImpl(); + ~DiskResourceImpl() override; + + // Implementation of ResourceInterface methods. + bool Reserve(uint64_t size) override; + void Discard(uint64_t size) override; + uint64_t GetTotal() override; + uint64_t GetUsed() override; + void Test_SetTotal(uint64_t test_total) override; + + private: + uint64_t total_; + std::atomic<uint64_t> used_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_RESOURCES_DISK_RESOURCE_IMPL_H_ diff --git a/chromium/components/reporting/storage/resources/memory_resource_impl.cc b/chromium/components/reporting/storage/resources/memory_resource_impl.cc new file mode 100644 index 00000000000..7d59caa9740 --- /dev/null +++ b/chromium/components/reporting/storage/resources/memory_resource_impl.cc @@ -0,0 +1,54 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/resources/memory_resource_impl.h" + +#include <atomic> +#include <cstdint> + +#include "base/check_op.h" +#include "base/no_destructor.h" + +namespace reporting { + +// TODO(b/159361496): Set total memory allowance based on the platform +// (or policy?). +MemoryResourceImpl::MemoryResourceImpl() + : total_(16u * 1024LLu * 1024LLu), // 16 MiB + used_(0) {} + +MemoryResourceImpl::~MemoryResourceImpl() = default; + +bool MemoryResourceImpl::Reserve(uint64_t size) { + uint64_t old_used = used_.fetch_add(size); + if (old_used + size > total_) { + used_.fetch_sub(size); + return false; + } + return true; +} + +void MemoryResourceImpl::Discard(uint64_t size) { + DCHECK_LE(size, used_.load()); + used_.fetch_sub(size); +} + +uint64_t MemoryResourceImpl::GetTotal() { + return total_; +} + +uint64_t MemoryResourceImpl::GetUsed() { + return used_.load(); +} + +void MemoryResourceImpl::Test_SetTotal(uint64_t test_total) { + total_ = test_total; +} + +ResourceInterface* GetMemoryResource() { + static base::NoDestructor<MemoryResourceImpl> memory; + return memory.get(); +} + +} // namespace reporting diff --git a/chromium/components/reporting/storage/resources/memory_resource_impl.h b/chromium/components/reporting/storage/resources/memory_resource_impl.h new file mode 100644 index 00000000000..c8cd965f87f --- /dev/null +++ b/chromium/components/reporting/storage/resources/memory_resource_impl.h @@ -0,0 +1,37 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_RESOURCES_MEMORY_RESOURCE_IMPL_H_ +#define COMPONENTS_REPORTING_STORAGE_RESOURCES_MEMORY_RESOURCE_IMPL_H_ + +#include <atomic> +#include <cstdint> + +#include "components/reporting/storage/resources/resource_interface.h" + +namespace reporting { + +// Interface to resources management by Storage module. +// Must be implemented by the caller base on the platform limitations. +// All APIs are non-blocking. +class MemoryResourceImpl : public ResourceInterface { + public: + MemoryResourceImpl(); + ~MemoryResourceImpl() override; + + // Implementation of ResourceInterface methods. + bool Reserve(uint64_t size) override; + void Discard(uint64_t size) override; + uint64_t GetTotal() override; + uint64_t GetUsed() override; + void Test_SetTotal(uint64_t test_total) override; + + private: + uint64_t total_; + std::atomic<uint64_t> used_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_RESOURCES_MEMORY_RESOURCE_IMPL_H_ diff --git a/chromium/components/reporting/storage/resources/resource_interface.cc b/chromium/components/reporting/storage/resources/resource_interface.cc new file mode 100644 index 00000000000..57e97c399f2 --- /dev/null +++ b/chromium/components/reporting/storage/resources/resource_interface.cc @@ -0,0 +1,34 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/resources/resource_interface.h" + +#include <cstdint> + +namespace reporting { + +ScopedReservation::ScopedReservation(uint64_t size, + ResourceInterface* resource_interface) + : resource_interface_(resource_interface) { + if (!resource_interface->Reserve(size)) { + return; + } + size_ = size; +} + +ScopedReservation::ScopedReservation(ScopedReservation&& other) + : resource_interface_(other.resource_interface_), + size_(std::move(other.size_)) {} + +bool ScopedReservation::reserved() const { + return size_.has_value(); +} + +ScopedReservation::~ScopedReservation() { + if (reserved()) { + resource_interface_->Discard(size_.value()); + } +} + +} // namespace reporting diff --git a/chromium/components/reporting/storage/resources/resource_interface.h b/chromium/components/reporting/storage/resources/resource_interface.h new file mode 100644 index 00000000000..a2b30299fbe --- /dev/null +++ b/chromium/components/reporting/storage/resources/resource_interface.h @@ -0,0 +1,78 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_RESOURCES_RESOURCE_INTERFACE_H_ +#define COMPONENTS_REPORTING_STORAGE_RESOURCES_RESOURCE_INTERFACE_H_ + +#include <cstdint> + +#include "base/optional.h" + +namespace reporting { + +// Interface to resources management by Storage module. +// Must be implemented by the caller base on the platform limitations. +// All APIs are non-blocking. +class ResourceInterface { + public: + virtual ~ResourceInterface() = default; + + // Needs to be called before attempting to allocate specified size. + // Returns true if requested amount can be allocated. + // After that the caller can actually allocate it or must call + // |Discard| if decided not to allocate. + virtual bool Reserve(uint64_t size) = 0; + + // Reverts reservation. + // Must be called after the specified amount is released. + virtual void Discard(uint64_t size) = 0; + + // Returns total amount. + virtual uint64_t GetTotal() = 0; + + // Returns current used amount. + virtual uint64_t GetUsed() = 0; + + // Test only: Sets non-default usage limit. + virtual void Test_SetTotal(uint64_t test_total) = 0; + + protected: + ResourceInterface() = default; +}; + +// Moveable RAII class used for scoped Reserve-Discard. +// +// Usage: +// { +// ScopedReservation reservation(1024u, GetMemoryResource()); +// if (!reservation.reserved()) { +// // Allocation failed. +// return; +// } +// // Allocation succeeded. +// ... +// } // Automatically discarded. +// +// Can be handed over to another owner. +class ScopedReservation { + public: + ScopedReservation(uint64_t size, ResourceInterface* resource_interface); + ScopedReservation(ScopedReservation&& other); + ScopedReservation(const ScopedReservation& other) = delete; + ScopedReservation& operator=(const ScopedReservation& other) = delete; + ~ScopedReservation(); + + bool reserved() const; + + private: + ResourceInterface* const resource_interface_; + base::Optional<uint64_t> size_; +}; + +ResourceInterface* GetMemoryResource(); +ResourceInterface* GetDiskResource(); + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_RESOURCES_RESOURCE_INTERFACE_H_ diff --git a/chromium/components/reporting/storage/resources/resource_interface_unittest.cc b/chromium/components/reporting/storage/resources/resource_interface_unittest.cc new file mode 100644 index 00000000000..90bddcf06ed --- /dev/null +++ b/chromium/components/reporting/storage/resources/resource_interface_unittest.cc @@ -0,0 +1,146 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <cstdint> + +#include "components/reporting/storage/resources/resource_interface.h" + +#include "base/task/post_task.h" +#include "base/task_runner.h" +#include "base/test/task_environment.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::Eq; + +namespace reporting { +namespace { + +class TestCallbackWaiter { + public: + TestCallbackWaiter() = default; + TestCallbackWaiter(const TestCallbackWaiter& other) = delete; + TestCallbackWaiter& operator=(const TestCallbackWaiter& other) = delete; + + void Attach() { + const size_t old_counter = counter_.fetch_add(1); + DCHECK_GT(old_counter, 0u) << "Cannot attach when already being released"; + } + + void Signal() { + const size_t old_counter = counter_.fetch_sub(1); + DCHECK_GT(old_counter, 0u) << "Already being released"; + if (old_counter == 1u) { + // Dropped last owner. + run_loop_.Quit(); + } + } + + void Wait() { + Signal(); // Rid of the constructor's ownership. + run_loop_.Run(); + } + + private: + std::atomic<size_t> counter_{1}; // Owned by constructor. + base::RunLoop run_loop_; +}; + +class ResourceInterfaceTest + : public ::testing::TestWithParam<ResourceInterface*> { + protected: + ResourceInterface* resource_interface() const { return GetParam(); } + + private: + base::test::TaskEnvironment task_environment_; +}; + +TEST_P(ResourceInterfaceTest, NestedReservationTest) { + uint64_t size = resource_interface()->GetTotal(); + while ((size / 2) > 0u) { + size /= 2; + EXPECT_TRUE(resource_interface()->Reserve(size)); + } + + for (; size < resource_interface()->GetTotal(); size *= 2) { + resource_interface()->Discard(size); + } + + EXPECT_THAT(resource_interface()->GetUsed(), Eq(0u)); +} + +TEST_P(ResourceInterfaceTest, SimultaneousReservationTest) { + uint64_t size = resource_interface()->GetTotal(); + + // Schedule reservations. + TestCallbackWaiter reserve_waiter; + while ((size / 2) > 0u) { + size /= 2; + reserve_waiter.Attach(); + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](size_t size, ResourceInterface* resource_interface, + TestCallbackWaiter* waiter) { + EXPECT_TRUE(resource_interface->Reserve(size)); + waiter->Signal(); + }, + size, resource_interface(), &reserve_waiter)); + } + reserve_waiter.Wait(); + + // Schedule discards. + TestCallbackWaiter discard_waiter; + for (; size < resource_interface()->GetTotal(); size *= 2) { + discard_waiter.Attach(); + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](size_t size, ResourceInterface* resource_interface, + TestCallbackWaiter* waiter) { + resource_interface->Discard(size); + waiter->Signal(); + }, + size, resource_interface(), &discard_waiter)); + } + discard_waiter.Wait(); + + EXPECT_THAT(resource_interface()->GetUsed(), Eq(0u)); +} + +TEST_P(ResourceInterfaceTest, SimultaneousScopedReservationTest) { + uint64_t size = resource_interface()->GetTotal(); + TestCallbackWaiter waiter; + while ((size / 2) > 0u) { + size /= 2; + waiter.Attach(); + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](size_t size, ResourceInterface* resource_interface, + TestCallbackWaiter* waiter) { + { ScopedReservation(size, resource_interface); } + waiter->Signal(); + }, + size, resource_interface(), &waiter)); + } + waiter.Wait(); + EXPECT_THAT(resource_interface()->GetUsed(), Eq(0u)); +} + +TEST_P(ResourceInterfaceTest, ReservationOverMaxTest) { + EXPECT_FALSE( + resource_interface()->Reserve(resource_interface()->GetTotal() + 1)); + EXPECT_TRUE(resource_interface()->Reserve(resource_interface()->GetTotal())); + resource_interface()->Discard(resource_interface()->GetTotal()); + EXPECT_THAT(resource_interface()->GetUsed(), Eq(0u)); +} + +INSTANTIATE_TEST_SUITE_P(VariousResources, + ResourceInterfaceTest, + testing::Values(GetMemoryResource(), + GetDiskResource())); + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage.cc b/chromium/components/reporting/storage/storage.cc new file mode 100644 index 00000000000..5c3eaf272e7 --- /dev/null +++ b/chromium/components/reporting/storage/storage.cc @@ -0,0 +1,647 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage.h" + +#include <cstdint> +#include <utility> +#include <vector> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/containers/flat_set.h" +#include "base/files/file.h" +#include "base/files/file_enumerator.h" +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/files/platform_file.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/task/task_traits.h" +#include "base/task_runner.h" +#include "base/threading/thread.h" +#include "base/threading/thread_task_runner_handle.h" +#include "components/reporting/encryption/encryption_module.h" +#include "components/reporting/encryption/verification.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/storage/storage_queue.h" +#include "components/reporting/storage/storage_uploader_interface.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/status_macros.h" +#include "components/reporting/util/statusor.h" +#include "components/reporting/util/task_runner_context.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" +#include "third_party/protobuf/src/google/protobuf/io/zero_copy_stream_impl.h" + +namespace reporting { + +namespace { + +// Parameters of individual queues. +// TODO(b/159352842): Deliver space and upload parameters from outside. + +constexpr base::FilePath::CharType kImmediateQueueSubdir[] = + FILE_PATH_LITERAL("Immediate"); +constexpr base::FilePath::CharType kImmediateQueuePrefix[] = + FILE_PATH_LITERAL("P_Immediate"); + +constexpr base::FilePath::CharType kFastBatchQueueSubdir[] = + FILE_PATH_LITERAL("FastBatch"); +constexpr base::FilePath::CharType kFastBatchQueuePrefix[] = + FILE_PATH_LITERAL("P_FastBatch"); +constexpr base::TimeDelta kFastBatchUploadPeriod = + base::TimeDelta::FromSeconds(1); + +constexpr base::FilePath::CharType kSlowBatchQueueSubdir[] = + FILE_PATH_LITERAL("SlowBatch"); +constexpr base::FilePath::CharType kSlowBatchQueuePrefix[] = + FILE_PATH_LITERAL("P_SlowBatch"); +constexpr base::TimeDelta kSlowBatchUploadPeriod = + base::TimeDelta::FromSeconds(20); + +constexpr base::FilePath::CharType kBackgroundQueueSubdir[] = + FILE_PATH_LITERAL("Background"); +constexpr base::FilePath::CharType kBackgroundQueuePrefix[] = + FILE_PATH_LITERAL("P_Background"); +constexpr base::TimeDelta kBackgroundQueueUploadPeriod = + base::TimeDelta::FromMinutes(1); + +constexpr base::FilePath::CharType kManualQueueSubdir[] = + FILE_PATH_LITERAL("Manual"); +constexpr base::FilePath::CharType kManualQueuePrefix[] = + FILE_PATH_LITERAL("P_Manual"); +constexpr base::TimeDelta kManualUploadPeriod = base::TimeDelta::Max(); + +constexpr base::FilePath::CharType kEncryptionKeyFilePrefix[] = + FILE_PATH_LITERAL("EncryptionKey."); +const int32_t kEncryptionKeyMaxFileSize = 256; + +// Returns vector of <priority, queue_options> for all expected queues in +// Storage. Queues are all located under the given root directory. +std::vector<std::pair<Priority, QueueOptions>> ExpectedQueues( + const StorageOptions& options) { + return { + std::make_pair(IMMEDIATE, QueueOptions(options) + .set_subdirectory(kImmediateQueueSubdir) + .set_file_prefix(kImmediateQueuePrefix)), + std::make_pair(FAST_BATCH, + QueueOptions(options) + .set_subdirectory(kFastBatchQueueSubdir) + .set_file_prefix(kFastBatchQueuePrefix) + .set_upload_period(kFastBatchUploadPeriod)), + std::make_pair(SLOW_BATCH, + QueueOptions(options) + .set_subdirectory(kSlowBatchQueueSubdir) + .set_file_prefix(kSlowBatchQueuePrefix) + .set_upload_period(kSlowBatchUploadPeriod)), + std::make_pair(BACKGROUND_BATCH, + QueueOptions(options) + .set_subdirectory(kBackgroundQueueSubdir) + .set_file_prefix(kBackgroundQueuePrefix) + .set_upload_period(kBackgroundQueueUploadPeriod)), + std::make_pair(MANUAL_BATCH, QueueOptions(options) + .set_subdirectory(kManualQueueSubdir) + .set_file_prefix(kManualQueuePrefix) + .set_upload_period(kManualUploadPeriod)), + }; +} + +} // namespace + +// Uploader interface adaptor for individual queue. +class Storage::QueueUploaderInterface : public UploaderInterface { + public: + QueueUploaderInterface(Priority priority, + std::unique_ptr<UploaderInterface> storage_interface) + : priority_(priority), storage_interface_(std::move(storage_interface)) {} + + // Factory method. + static StatusOr<std::unique_ptr<UploaderInterface>> ProvideUploader( + Priority priority, + Storage* storage) { + ASSIGN_OR_RETURN( + std::unique_ptr<UploaderInterface> uploader, + storage->start_upload_cb_.Run( + priority, EncryptionModule::is_enabled() && + storage->encryption_module_->need_encryption_key())); + return std::make_unique<QueueUploaderInterface>(priority, + std::move(uploader)); + } + + void ProcessRecord(EncryptedRecord encrypted_record, + base::OnceCallback<void(bool)> processed_cb) override { + // Update sequencing information: add Priority. + SequencingInformation* const sequencing_info = + encrypted_record.mutable_sequencing_information(); + sequencing_info->set_priority(priority_); + storage_interface_->ProcessRecord(std::move(encrypted_record), + std::move(processed_cb)); + } + + void ProcessGap(SequencingInformation start, + uint64_t count, + base::OnceCallback<void(bool)> processed_cb) override { + // Update sequencing information: add Priority. + start.set_priority(priority_); + storage_interface_->ProcessGap(std::move(start), count, + std::move(processed_cb)); + } + + void Completed(Status final_status) override { + storage_interface_->Completed(final_status); + } + + private: + const Priority priority_; + const std::unique_ptr<UploaderInterface> storage_interface_; +}; + +class Storage::KeyInStorage { + public: + explicit KeyInStorage(base::StringPiece signature_verification_public_key, + const base::FilePath& directory) + : verifier_(signature_verification_public_key), directory_(directory) {} + ~KeyInStorage() = default; + + // Uploads signed encryption key to a file with an |index| >= + // |next_key_file_index_|. Returns status in case of any error. If succeeds, + // removes all files with lower indexes (if any). Called every time encryption + // key is updated. + Status UploadKeyFile(const SignedEncryptionInfo& signed_encryption_key) { + // Atomically reserve file index (none else will get the same index). + uint64_t new_file_index = next_key_file_index_.fetch_add(1); + // Write into file. + RETURN_IF_ERROR(WriteKeyInfoFile(new_file_index, signed_encryption_key)); + + // Enumerate data files and delete all files with lower index. + RemoveKeyFilesWithLowerIndexes(new_file_index); + return Status::StatusOK(); + } + + // Locates and downloads the latest valid enumeration keys file. + // Atomically sets |next_key_file_index_| to the a value larger than any found + // file. Returns key and key id pair, or error status (NOT_FOUND if no valid + // file has been found). Called once during initialization only. + StatusOr<std::pair<std::string, Encryptor::PublicKeyId>> DownloadKeyFile() { + // Make sure the assigned directory exists. + base::File::Error error; + if (!base::CreateDirectoryAndGetError(directory_, &error)) { + return Status( + error::UNAVAILABLE, + base::StrCat( + {"Storage directory '", directory_.MaybeAsASCII(), + "' does not exist, error=", base::File::ErrorToString(error)})); + } + + // Enumerate possible key files, collect the ones that have valid name, + // set next_key_file_index_ to a value that is definitely not used. + base::flat_set<base::FilePath> all_key_files; + base::flat_map<uint64_t, base::FilePath> found_key_files; + EnumerateKeyFiles(&all_key_files, &found_key_files); + + // Try to unserialize the key from each found file (latest first). + auto signed_encryption_key_result = LocateValidKeyAndParse(found_key_files); + + // If not found, return error. + if (!signed_encryption_key_result.has_value()) { + return Status(error::NOT_FOUND, "No valid encryption key found"); + } + + // Found and validated, delete all other files. + for (const auto& full_name : all_key_files) { + if (full_name == signed_encryption_key_result.value().first) { + continue; // This file is used. + } + base::DeleteFile(full_name); // Ignore errors, if any. + } + + // Return the key. + return std::make_pair( + signed_encryption_key_result.value().second.public_asymmetric_key(), + signed_encryption_key_result.value().second.public_key_id()); + } + + Status VerifySignature(const SignedEncryptionInfo& signed_encryption_key) { + if (signed_encryption_key.public_asymmetric_key().size() != + X25519_PUBLIC_VALUE_LEN) { + return Status{error::FAILED_PRECONDITION, "Key size mismatch"}; + } + char value_to_verify[sizeof(Encryptor::PublicKeyId) + + X25519_PUBLIC_VALUE_LEN]; + const Encryptor::PublicKeyId public_key_id = + signed_encryption_key.public_key_id(); + memcpy(value_to_verify, &public_key_id, sizeof(Encryptor::PublicKeyId)); + memcpy(value_to_verify + sizeof(Encryptor::PublicKeyId), + signed_encryption_key.public_asymmetric_key().data(), + X25519_PUBLIC_VALUE_LEN); + return verifier_.Verify( + std::string(value_to_verify, sizeof(value_to_verify)), + signed_encryption_key.signature()); + } + + private: + // Writes key into file. Called during key upload. + Status WriteKeyInfoFile(uint64_t new_file_index, + const SignedEncryptionInfo& signed_encryption_key) { + base::FilePath key_file_path = + directory_.Append(kEncryptionKeyFilePrefix) + .AddExtensionASCII(base::NumberToString(new_file_index)); + base::File key_file(key_file_path, + base::File::FLAG_OPEN_ALWAYS | base::File::FLAG_APPEND); + if (!key_file.IsValid()) { + return Status( + error::DATA_LOSS, + base::StrCat({"Cannot open key file='", key_file_path.MaybeAsASCII(), + "' for append"})); + } + std::string serialized_key; + if (!signed_encryption_key.SerializeToString(&serialized_key) || + serialized_key.empty()) { + return Status(error::DATA_LOSS, + base::StrCat({"Failed to seralize key into file='", + key_file_path.MaybeAsASCII(), "'"})); + } + const int32_t write_result = key_file.Write( + /*offset=*/0, serialized_key.data(), serialized_key.size()); + if (write_result < 0) { + return Status( + error::DATA_LOSS, + base::StrCat({"File write error=", + key_file.ErrorToString(key_file.GetLastFileError()), + " file=", key_file_path.MaybeAsASCII()})); + } + if (static_cast<size_t>(write_result) != serialized_key.size()) { + return Status(error::DATA_LOSS, + base::StrCat({"Failed to seralize key into file='", + key_file_path.MaybeAsASCII(), "'"})); + } + return Status::StatusOK(); + } + + // Enumerates key files and deletes those with index lower than + // |new_file_index|. Called during key upload. + void RemoveKeyFilesWithLowerIndexes(uint64_t new_file_index) { + base::flat_set<base::FilePath> key_files_to_remove; + base::FileEnumerator dir_enum( + directory_, + /*recursive=*/false, base::FileEnumerator::FILES, + base::StrCat({kEncryptionKeyFilePrefix, FILE_PATH_LITERAL("*")})); + base::FilePath full_name; + while (full_name = dir_enum.Next(), !full_name.empty()) { + const auto result = key_files_to_remove.emplace(full_name); + if (!result.second) { + // Duplicate file name. Should not happen. + continue; + } + const auto extension = full_name.Extension(); + if (extension.empty()) { + // Should not happen, will remove this file. + continue; + } + uint64_t file_index = 0; + if (!base::StringToUint64(extension.substr(1), &file_index)) { + // Bad extension - not a number. Should not happen, will remove this + // file. + continue; + } + if (file_index < new_file_index) { + // Lower index file, will remove it. + continue; + } + // Keep this file - drop it from erase list. + key_files_to_remove.erase(result.first); + } + // Delete all files assigned for deletion. + for (const auto& full_name : key_files_to_remove) { + base::DeleteFile(full_name); // Ignore errors, if any. + } + } + + // Enumerates possible key files, collects the ones that have valid name, + // sets next_key_file_index_ to a value that is definitely not used. + // Called once, during initialization. + void EnumerateKeyFiles( + base::flat_set<base::FilePath>* all_key_files, + base::flat_map<uint64_t, base::FilePath>* found_key_files) { + base::FileEnumerator dir_enum( + directory_, + /*recursive=*/false, base::FileEnumerator::FILES, + base::StrCat({kEncryptionKeyFilePrefix, FILE_PATH_LITERAL("*")})); + base::FilePath full_name; + while (full_name = dir_enum.Next(), !full_name.empty()) { + if (!all_key_files->emplace(full_name).second) { + // Duplicate file name. Should not happen. + continue; + } + const auto extension = full_name.Extension(); + if (extension.empty()) { + // Should not happen. + continue; + } + uint64_t file_index = 0; + bool success = base::StringToUint64(extension.substr(1), &file_index); + if (!success) { + // Bad extension - not a number. Should not happen (file is corrupt). + continue; + } + if (!found_key_files->emplace(file_index, full_name).second) { + // Duplicate extension (e.g., 01 and 001). Should not happen (file is + // corrupt). + continue; + } + // Set 'next_key_file_index_' to a number which is definitely not used. + if (next_key_file_index_.load() <= file_index) { + next_key_file_index_.store(file_index + 1); + } + } + } + + // Enumerates found key files and locates one with the highest index and + // valid key. Returns pair of file name and loaded signed key proto. + // Called once, during initialization. + base::Optional<std::pair<base::FilePath, SignedEncryptionInfo>> + LocateValidKeyAndParse( + const base::flat_map<uint64_t, base::FilePath>& found_key_files) { + // Try to unserialize the key from each found file (latest first). + for (auto key_file_it = found_key_files.rbegin(); + key_file_it != found_key_files.rend(); ++key_file_it) { + base::File key_file(key_file_it->second, + base::File::FLAG_OPEN | base::File::FLAG_READ); + if (!key_file.IsValid()) { + continue; // Could not open. + } + + SignedEncryptionInfo signed_encryption_key; + { + const auto key_file_buffer = + std::make_unique<char[]>(kEncryptionKeyMaxFileSize); + const int32_t read_result = key_file.Read( + /*offset=*/0, key_file_buffer.get(), kEncryptionKeyMaxFileSize); + if (read_result < 0) { + LOG(WARNING) << "File read error=" + << key_file.ErrorToString(key_file.GetLastFileError()) + << " " << key_file_it->second.MaybeAsASCII(); + continue; // File read error. + } + if (read_result == 0 || read_result >= kEncryptionKeyMaxFileSize) { + continue; // Unexpected file size. + } + google::protobuf::io::ArrayInputStream key_stream( // Zero-copy stream. + key_file_buffer.get(), read_result); + if (!signed_encryption_key.ParseFromZeroCopyStream(&key_stream)) { + LOG(WARNING) << "Failed to parse key file, full_name='" + << key_file_it->second.MaybeAsASCII() << "'"; + continue; + } + } + + // Parsed successfully. Verify signature of the whole "id"+"key" string. + const auto signature_verification_status = + VerifySignature(signed_encryption_key); + if (!signature_verification_status.ok()) { + LOG(WARNING) << "Loaded key failed verification, status=" + << signature_verification_status << ", full_name='" + << key_file_it->second.MaybeAsASCII() << "'"; + continue; + } + + // Validated successfully. Return file name and signed key proto. + return std::make_pair(key_file_it->second, signed_encryption_key); + } + + // Not found, return error. + return base::nullopt; + } + + // Index of the file to serialize the signed key to. + // Initialized to the next available number or 0, if none present. + // Every time a new key is received, it is stored in a file with the next + // index; however, any file found with the matching signature can be used + // to successfully encrypt records and for the server to then decrypt them. + std::atomic<uint64_t> next_key_file_index_{0}; + + SignatureVerifier verifier_; + + const base::FilePath directory_; +}; + +void Storage::Create( + const StorageOptions& options, + UploaderInterface::StartCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module, + base::OnceCallback<void(StatusOr<scoped_refptr<Storage>>)> completion_cb) { + // Initialize Storage object, populating all the queues. + class StorageInitContext + : public TaskRunnerContext<StatusOr<scoped_refptr<Storage>>> { + public: + StorageInitContext( + const std::vector<std::pair<Priority, QueueOptions>>& queues_options, + scoped_refptr<Storage> storage, + base::OnceCallback<void(StatusOr<scoped_refptr<Storage>>)> callback) + : TaskRunnerContext<StatusOr<scoped_refptr<Storage>>>( + std::move(callback), + base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::BEST_EFFORT, base::MayBlock()})), + queues_options_(queues_options), + storage_(std::move(storage)) {} + + private: + // Context can only be deleted by calling Response method. + ~StorageInitContext() override { DCHECK_EQ(count_, 0); } + + void OnStart() override { + CheckOnValidSequence(); + + // Locate the latest signed_encryption_key file with matching key + // signature after deserialization. + const auto download_key_result = + storage_->key_in_storage_->DownloadKeyFile(); + if (!download_key_result.ok()) { + // Key not found or corrupt. Proceed with queues creation directly. + EncryptionSetUp(download_key_result.status()); + return; + } + + // Key found, verified and downloaded. + storage_->encryption_module_->UpdateAsymmetricKey( + download_key_result.ValueOrDie().first, + download_key_result.ValueOrDie().second, + base::BindOnce(&StorageInitContext::ScheduleEncryptionSetUp, + base::Unretained(this))); + } + + void ScheduleEncryptionSetUp(Status status) { + Schedule(&StorageInitContext::EncryptionSetUp, base::Unretained(this), + status); + } + + void EncryptionSetUp(Status status) { + CheckOnValidSequence(); + + if (status.ok()) { + // Encryption key has been found and set up. Must be available now. + DCHECK(storage_->encryption_module_->has_encryption_key()); + } else { + if (EncryptionModule::is_enabled()) { + // Encryptor enabled - we cannot proceed with no keys. + // Send Upload with need_encryption_key flag and no records. + StatusOr<std::unique_ptr<UploaderInterface>> uploader = + storage_->start_upload_cb_.Run( + /*priority=*/MANUAL_BATCH, // Any priority would do. + /*need_encryption_key=*/true); + if (!uploader.ok()) { + Response(uploader.status()); + return; + } + uploader.ValueOrDie()->Completed(Status::StatusOK()); + // Continue initialization without waiting for it to respond. + // Until the response arrives, we will reject Enqueues. + } + } + + // Construct all queues. + count_ = queues_options_.size(); + for (const auto& queue_options : queues_options_) { + StorageQueue::Create( + /*options=*/queue_options.second, + // Note: the callback below belongs to the Queue and does not + // outlive Storage. + base::BindRepeating(&QueueUploaderInterface::ProvideUploader, + /*priority=*/queue_options.first, + base::Unretained(storage_.get())), + storage_->encryption_module_, + base::BindOnce(&StorageInitContext::ScheduleAddQueue, + base::Unretained(this), + /*priority=*/queue_options.first)); + } + } + + void ScheduleAddQueue( + Priority priority, + StatusOr<scoped_refptr<StorageQueue>> storage_queue_result) { + Schedule(&StorageInitContext::AddQueue, base::Unretained(this), priority, + std::move(storage_queue_result)); + } + + void AddQueue(Priority priority, + StatusOr<scoped_refptr<StorageQueue>> storage_queue_result) { + CheckOnValidSequence(); + if (storage_queue_result.ok()) { + auto add_result = storage_->queues_.emplace( + priority, storage_queue_result.ValueOrDie()); + DCHECK(add_result.second); + } else { + LOG(ERROR) << "Could not create queue, priority=" << priority + << ", status=" << storage_queue_result.status(); + if (final_status_.ok()) { + final_status_ = storage_queue_result.status(); + } + } + DCHECK_GT(count_, 0); + if (--count_ > 0) { + return; + } + if (!final_status_.ok()) { + Response(final_status_); + return; + } + Response(std::move(storage_)); + } + + const std::vector<std::pair<Priority, QueueOptions>> queues_options_; + scoped_refptr<Storage> storage_; + int32_t count_ = 0; + Status final_status_; + }; + + // Create Storage object. + // Cannot use base::MakeRefCounted<Storage>, because constructor is private. + scoped_refptr<Storage> storage = base::WrapRefCounted( + new Storage(options, encryption_module, std::move(start_upload_cb))); + + // Asynchronously run initialization. + Start<StorageInitContext>(ExpectedQueues(storage->options_), + std::move(storage), std::move(completion_cb)); +} + +Storage::Storage(const StorageOptions& options, + scoped_refptr<EncryptionModule> encryption_module, + UploaderInterface::StartCb start_upload_cb) + : options_(options), + encryption_module_(encryption_module), + key_in_storage_(std::make_unique<KeyInStorage>( + options.signature_verification_public_key(), + options.directory())), + start_upload_cb_(std::move(start_upload_cb)) {} + +Storage::~Storage() = default; + +void Storage::Write(Priority priority, + Record record, + base::OnceCallback<void(Status)> completion_cb) { + // Note: queues_ never change after initialization is finished, so there is + // no need to protect or serialize access to it. + ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(scoped_refptr<StorageQueue> queue, + completion_cb, GetQueue(priority)); + queue->Write(std::move(record), std::move(completion_cb)); +} + +void Storage::Confirm(Priority priority, + base::Optional<int64_t> seq_number, + bool force, + base::OnceCallback<void(Status)> completion_cb) { + // Note: queues_ never change after initialization is finished, so there is + // no need to protect or serialize access to it. + ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(scoped_refptr<StorageQueue> queue, + completion_cb, GetQueue(priority)); + queue->Confirm(seq_number, force, std::move(completion_cb)); +} + +Status Storage::Flush(Priority priority) { + // Note: queues_ never change after initialization is finished, so there is + // no need to protect or serialize access to it. + ASSIGN_OR_RETURN(scoped_refptr<StorageQueue> queue, GetQueue(priority)); + queue->Flush(); + return Status::StatusOK(); +} + +void Storage::UpdateEncryptionKey(SignedEncryptionInfo signed_encryption_key) { + // Verify received key signature. Bail out if failed. + const auto signature_verification_status = + key_in_storage_->VerifySignature(signed_encryption_key); + if (!signature_verification_status.ok()) { + LOG(WARNING) << "Key failed verification, status=" + << signature_verification_status; + return; + } + + // Assign the received key to encryption module. + encryption_module_->UpdateAsymmetricKey( + signed_encryption_key.public_asymmetric_key(), + signed_encryption_key.public_key_id(), base::BindOnce([](Status status) { + if (!status.ok()) { + LOG(WARNING) << "Encryption key update failed, status=" << status; + return; + } + // Encryption key updated successfully. + })); + + // Serialize whole signed_encryption_key to a new file, discard the old + // one(s). + const Status status = key_in_storage_->UploadKeyFile(signed_encryption_key); + LOG_IF(ERROR, !status.ok()) << "Failed to upload the new encription key."; +} + +StatusOr<scoped_refptr<StorageQueue>> Storage::GetQueue(Priority priority) { + auto it = queues_.find(priority); + if (it == queues_.end()) { + return Status( + error::NOT_FOUND, + base::StrCat({"Undefined priority=", base::NumberToString(priority)})); + } + return it->second; +} + +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage.h b/chromium/components/reporting/storage/storage.h new file mode 100644 index 00000000000..d7aa8feba6c --- /dev/null +++ b/chromium/components/reporting/storage/storage.h @@ -0,0 +1,120 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_STORAGE_H_ +#define COMPONENTS_REPORTING_STORAGE_STORAGE_H_ + +#include <map> +#include <memory> +#include <string> +#include <utility> + +#include "base/callback.h" +#include "base/containers/flat_map.h" +#include "base/files/file_path.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "base/strings/string_piece.h" +#include "components/reporting/encryption/encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/storage/storage_queue.h" +#include "components/reporting/storage/storage_uploader_interface.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// Storage represents the data to be collected, stored persistently and uploaded +// according to the priority. +class Storage : public base::RefCountedThreadSafe<Storage> { + public: + // Creates Storage instance, and returns it with the completion callback. + static void Create( + const StorageOptions& options, + UploaderInterface::StartCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module, + base::OnceCallback<void(StatusOr<scoped_refptr<Storage>>)> completion_cb); + + // Wraps and serializes Record (taking ownership of it), encrypts and writes + // the resulting blob into the Storage (the last file of it) according to the + // priority with the next sequencing id assigned. If file is going to + // become too large, it is closed and new file is created. + void Write(Priority priority, + Record record, + base::OnceCallback<void(Status)> completion_cb); + + // Confirms acceptance of the records according to the priority up to + // |sequencing_id| (inclusively). All records with sequeincing ids <= this + // one can be removed from the Storage, and can no longer be uploaded. + // If |force| is false (which is used in most cases), |sequencing_id| is + // only accepted if no higher ids were confirmed before; otherwise it is + // accepted unconditionally. + void Confirm(Priority priority, + base::Optional<int64_t> sequencing_id, + bool force, + base::OnceCallback<void(Status)> completion_cb); + + // Initiates upload of collected records according to the priority. + // Called usually for a queue with an infinite or very large upload period. + // Multiple |Flush| calls can safely run in parallel. + // Returns error if cannot start upload. + Status Flush(Priority priority); + + // If the server attached signed encryption key to the response, it needs to + // be paased here. + void UpdateEncryptionKey(SignedEncryptionInfo signed_encryption_key); + + Storage(const Storage& other) = delete; + Storage& operator=(const Storage& other) = delete; + + protected: + virtual ~Storage(); + + private: + friend class base::RefCountedThreadSafe<Storage>; + + // Private bridge class. + class QueueUploaderInterface; + + // Private helper class for key upload/download to the file system. + class KeyInStorage; + + // Private constructor, to be called by Create factory method only. + // Queues need to be added afterwards. + Storage(const StorageOptions& options, + scoped_refptr<EncryptionModule> encryption_module, + UploaderInterface::StartCb start_upload_cb); + + // Initializes the object by adding all queues for all priorities. + // Must be called once and only once after construction. + // Returns OK or error status, if anything failed to initialize. + Status Init(); + + // Helper method that selects queue by priority. Returns error + // if priority does not match any queue. + // Note: queues_ never change after initialization is finished, so there is no + // need to protect or serialize access to it. + StatusOr<scoped_refptr<StorageQueue>> GetQueue(Priority priority); + + // Immutable options, stored at the time of creation. + const StorageOptions options_; + + // Encryption module. + scoped_refptr<EncryptionModule> encryption_module_; + + // Internal key management module. + std::unique_ptr<KeyInStorage> key_in_storage_; + + // Map priority->StorageQueue. + base::flat_map<Priority, scoped_refptr<StorageQueue>> queues_; + + // Upload provider callback. + const UploaderInterface::StartCb start_upload_cb_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_STORAGE_H_ diff --git a/chromium/components/reporting/storage/storage_configuration.cc b/chromium/components/reporting/storage/storage_configuration.cc new file mode 100644 index 00000000000..df4fc36b068 --- /dev/null +++ b/chromium/components/reporting/storage/storage_configuration.cc @@ -0,0 +1,15 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage_configuration.h" + +namespace reporting { + +StorageOptions::StorageOptions() = default; +StorageOptions::StorageOptions(const StorageOptions& options) = default; +StorageOptions& StorageOptions::operator=(const StorageOptions& options) = + default; +StorageOptions::~StorageOptions() = default; + +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_configuration.h b/chromium/components/reporting/storage/storage_configuration.h new file mode 100644 index 00000000000..834d3ff0338 --- /dev/null +++ b/chromium/components/reporting/storage/storage_configuration.h @@ -0,0 +1,145 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_STORAGE_CONFIGURATION_H_ +#define COMPONENTS_REPORTING_STORAGE_STORAGE_CONFIGURATION_H_ + +#include <string> + +#include "base/files/file_path.h" +#include "base/strings/string_piece.h" +#include "base/time/time.h" + +namespace reporting { + +// Storage options class allowing to set parameters individually, e.g.: +// Storage::Create(Options() +// .set_directory("/var/cache/reporting") +// .set_max_record_size(4 * 1024u) +// .set_max_total_files_size(64 * 1024u * 1024u) +// .set_max_total_memory_size(256 * 1024u), +// callback); +class StorageOptions { + public: + StorageOptions(); + StorageOptions(const StorageOptions& options); + StorageOptions& operator=(const StorageOptions& options); + ~StorageOptions(); + StorageOptions& set_directory(const base::FilePath& directory) { + directory_ = directory; + return *this; + } + StorageOptions& set_signature_verification_public_key( + base::StringPiece signature_verification_public_key) { + signature_verification_public_key_ = + std::string(signature_verification_public_key); + return *this; + } + StorageOptions& set_max_record_size(size_t max_record_size) { + max_record_size_ = max_record_size; + return *this; + } + StorageOptions& set_max_total_files_size(uint64_t max_total_files_size) { + max_total_files_size_ = max_total_files_size; + return *this; + } + StorageOptions& set_max_total_memory_size(uint64_t max_total_memory_size) { + max_total_memory_size_ = max_total_memory_size; + return *this; + } + StorageOptions& set_single_file_size(uint64_t single_file_size) { + single_file_size_ = single_file_size; + return *this; + } + const base::FilePath& directory() const { return directory_; } + base::StringPiece signature_verification_public_key() const { + return signature_verification_public_key_; + } + size_t max_record_size() const { return max_record_size_; } + uint64_t max_total_files_size() const { return max_total_files_size_; } + uint64_t max_total_memory_size() const { return max_total_memory_size_; } + uint64_t single_file_size() const { return single_file_size_; } + + private: + // Subdirectory of the location assigned for this Storage. + base::FilePath directory_; + + // Public key for signature verification when encryption key + // is delivered to Storage. + std::string signature_verification_public_key_; + + // Maximum record size. + size_t max_record_size_ = 1 * 1024LL * 1024LL; // 1 MiB + + // Maximum total size of all files in all queues. + uint64_t max_total_files_size_ = 64 * 1024LL * 1024LL; // 64 MiB + + // Maximum memory usage (reading buffers). + uint64_t max_total_memory_size_ = 4 * 1024LL * 1024LL; // 4 MiB + + // Cut-off size of an individual file in all queues. + // When file exceeds this size, the new file is created + // for further records. Note that each file must have at least + // one record before it is closed, regardless of that record size. + uint64_t single_file_size_ = 1 * 1024LL * 1024LL; // 1 MiB +}; + +// Single queue options class allowing to set parameters individually, e.g.: +// StorageQueue::Create(QueueOptions(storage_options) +// .set_subdirectory("reporting") +// .set_file_prefix(FILE_PATH_LITERAL("p00000001")), +// callback); +// storage_options must outlive QueueOptions. +class QueueOptions { + public: + explicit QueueOptions(const StorageOptions& storage_options) + : storage_options_(storage_options) {} + QueueOptions(const QueueOptions& options) = default; + // QueueOptions& operator=(const QueueOptions& options) = default; + QueueOptions& set_subdirectory( + const base::FilePath::StringType& subdirectory) { + directory_ = storage_options_.directory().Append(subdirectory); + return *this; + } + QueueOptions& set_file_prefix(const base::FilePath::StringType& file_prefix) { + file_prefix_ = file_prefix; + return *this; + } + QueueOptions& set_upload_period(base::TimeDelta upload_period) { + upload_period_ = upload_period; + return *this; + } + const base::FilePath& directory() const { return directory_; } + const base::FilePath::StringType& file_prefix() const { return file_prefix_; } + size_t max_record_size() const { return storage_options_.max_record_size(); } + size_t max_total_files_size() const { + return storage_options_.max_total_files_size(); + } + size_t max_total_memory_size() const { + return storage_options_.max_total_memory_size(); + } + uint64_t single_file_size() const { + return storage_options_.single_file_size(); + } + base::TimeDelta upload_period() const { return upload_period_; } + + private: + // Whole storage options, which this queue options are based on. + const StorageOptions& storage_options_; + + // Subdirectory of the Storage location assigned for this StorageQueue. + base::FilePath directory_; + // Prefix of data files assigned for this StorageQueue. + base::FilePath::StringType file_prefix_; + // Time period the data is uploaded with. + // If 0, uploaded immediately after a new record is stored + // (this setting is intended for the immediate priority). + // Can be set to infinity - in that case Flush() is expected to be + // called from time to time. + base::TimeDelta upload_period_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_STORAGE_CONFIGURATION_H_ diff --git a/chromium/components/reporting/storage/storage_module.cc b/chromium/components/reporting/storage/storage_module.cc new file mode 100644 index 00000000000..d938b27b73e --- /dev/null +++ b/chromium/components/reporting/storage/storage_module.cc @@ -0,0 +1,79 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage_module.h" + +#include <memory> +#include <utility> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/containers/span.h" +#include "base/memory/ptr_util.h" +#include "components/reporting/encryption/encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/storage/storage.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/storage/storage_module_interface.h" +#include "components/reporting/storage/storage_uploader_interface.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +StorageModule::StorageModule() = default; + +StorageModule::~StorageModule() = default; + +void StorageModule::AddRecord(Priority priority, + Record record, + base::OnceCallback<void(Status)> callback) { + storage_->Write(priority, std::move(record), std::move(callback)); +} + +void StorageModule::ReportSuccess(SequencingInformation sequencing_information, + bool force) { + storage_->Confirm( + sequencing_information.priority(), sequencing_information.sequencing_id(), + force, base::BindOnce([](Status status) { + if (!status.ok()) { + LOG(ERROR) << "Unable to confirm record deletion: " << status; + } + })); +} + +void StorageModule::UpdateEncryptionKey( + SignedEncryptionInfo signed_encryption_key) { + storage_->UpdateEncryptionKey(std::move(signed_encryption_key)); +} + +// static +void StorageModule::Create( + const StorageOptions& options, + UploaderInterface::StartCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module, + base::OnceCallback<void(StatusOr<scoped_refptr<StorageModuleInterface>>)> + callback) { + scoped_refptr<StorageModule> instance = + // Cannot base::MakeRefCounted, since constructor is protected. + base::WrapRefCounted(new StorageModule()); + Storage::Create( + options, start_upload_cb, encryption_module, + base::BindOnce( + [](scoped_refptr<StorageModule> instance, + base::OnceCallback<void( + StatusOr<scoped_refptr<StorageModuleInterface>>)> callback, + StatusOr<scoped_refptr<Storage>> storage) { + if (!storage.ok()) { + std::move(callback).Run(storage.status()); + return; + } + instance->storage_ = std::move(storage.ValueOrDie()); + std::move(callback).Run(std::move(instance)); + }, + std::move(instance), std::move(callback))); +} + +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_module.h b/chromium/components/reporting/storage/storage_module.h new file mode 100644 index 00000000000..eecb0ff64d2 --- /dev/null +++ b/chromium/components/reporting/storage/storage_module.h @@ -0,0 +1,74 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_STORAGE_MODULE_H_ +#define COMPONENTS_REPORTING_STORAGE_STORAGE_MODULE_H_ + +#include <utility> + +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "components/reporting/encryption/encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/storage/storage.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/storage/storage_module_interface.h" +#include "components/reporting/storage/storage_uploader_interface.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +class StorageModule : public StorageModuleInterface { + public: + // Factory method creates |StorageModule| object. + static void Create( + const StorageOptions& options, + UploaderInterface::StartCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module, + base::OnceCallback<void(StatusOr<scoped_refptr<StorageModuleInterface>>)> + callback); + + StorageModule(const StorageModule& other) = delete; + StorageModule& operator=(const StorageModule& other) = delete; + + // AddRecord will add |record| (taking ownership) to the |StorageModule| + // according to the provided |priority|. On completion, |callback| will be + // called. + void AddRecord(Priority priority, + Record record, + base::OnceCallback<void(Status)> callback) override; + + // Once a record has been successfully uploaded, the sequencing information + // can be passed back to the StorageModule here for record deletion. + // If |force| is false (which is used in most cases), |sequencing_information| + // only affects Storage if no higher sequeincing was confirmed before; + // otherwise it is accepted unconditionally. + void ReportSuccess(SequencingInformation sequencing_information, + bool force) override; + + // If the server attached signed encryption key to the response, it needs to + // be paased here. + void UpdateEncryptionKey(SignedEncryptionInfo signed_encryption_key) override; + + protected: + // Constructor can only be called by |Create| factory method. + StorageModule(); + + // Refcounted object must have destructor declared protected or private. + ~StorageModule() override; + + private: + friend base::RefCountedThreadSafe<StorageModule>; + + // Storage backend (currently only Storage). + // TODO(b/160334561): make it a pluggable interface. + scoped_refptr<Storage> storage_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_STORAGE_MODULE_H_ diff --git a/chromium/components/reporting/storage/storage_module_interface.cc b/chromium/components/reporting/storage/storage_module_interface.cc new file mode 100644 index 00000000000..882a0055a9f --- /dev/null +++ b/chromium/components/reporting/storage/storage_module_interface.cc @@ -0,0 +1,12 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage_module_interface.h" + +namespace reporting { + +StorageModuleInterface::StorageModuleInterface() = default; +StorageModuleInterface::~StorageModuleInterface() = default; + +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_module_interface.h b/chromium/components/reporting/storage/storage_module_interface.h new file mode 100644 index 00000000000..f942d3b7350 --- /dev/null +++ b/chromium/components/reporting/storage/storage_module_interface.h @@ -0,0 +1,59 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_STORAGE_MODULE_INTERFACE_H_ +#define COMPONENTS_REPORTING_STORAGE_STORAGE_MODULE_INTERFACE_H_ + +#include <utility> + +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/util/status.h" + +namespace reporting { + +class StorageModuleInterface + : public base::RefCountedThreadSafe<StorageModuleInterface> { + public: + StorageModuleInterface(const StorageModuleInterface& other) = delete; + StorageModuleInterface& operator=(const StorageModuleInterface& other) = + delete; + + // AddRecord will add |record| (taking ownership) to the + // |StorageModuleInterface| according to the provided |priority|. On + // completion, |callback| is called. + virtual void AddRecord(Priority priority, + Record record, + base::OnceCallback<void(Status)> callback) = 0; + + // Once a record has been successfully uploaded, the sequencing information + // can be passed back to the StorageModuleInterface here for record deletion. + // If |force| is false (which is used in most cases), |sequencing_information| + // only affects Storage if no higher sequeincing was confirmed before; + // otherwise it is accepted unconditionally. + virtual void ReportSuccess(SequencingInformation sequencing_information, + bool force) = 0; + + // If the server attached signed encryption key to the response, it needs to + // be paased here. + virtual void UpdateEncryptionKey( + SignedEncryptionInfo signed_encryption_key) = 0; + + protected: + // Constructor can only be called by |Create| factory method. + StorageModuleInterface(); + + // Refcounted object must have destructor declared protected or private. + virtual ~StorageModuleInterface(); + + private: + friend base::RefCountedThreadSafe<StorageModuleInterface>; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_STORAGE_MODULE_INTERFACE_H_ diff --git a/chromium/components/reporting/storage/storage_queue.cc b/chromium/components/reporting/storage/storage_queue.cc new file mode 100644 index 00000000000..c2007d47ba8 --- /dev/null +++ b/chromium/components/reporting/storage/storage_queue.cc @@ -0,0 +1,1563 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage_queue.h" + +#include <algorithm> +#include <cstring> +#include <iterator> +#include <list> +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/containers/flat_set.h" +#include "base/files/file.h" +#include "base/files/file_enumerator.h" +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/hash/hash.h" +#include "base/memory/ptr_util.h" +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/rand_util.h" +#include "base/sequence_checker.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_piece.h" +#include "base/strings/string_split.h" +#include "base/strings/string_util.h" +#include "base/task/post_task.h" +#include "base/task_runner.h" +#include "components/reporting/encryption/encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/storage/resources/resource_interface.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/storage/storage_uploader_interface.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/status_macros.h" +#include "components/reporting/util/statusor.h" +#include "components/reporting/util/task_runner_context.h" +#include "crypto/random.h" +#include "crypto/sha2.h" +#include "third_party/protobuf/src/google/protobuf/io/zero_copy_stream_impl_lite.h" + +namespace reporting { + +namespace { + +// Metadata file name prefix. +const base::FilePath::CharType METADATA_NAME[] = FILE_PATH_LITERAL("META"); + +// The size in bytes that all files and records are rounded to (for privacy: +// make it harder to differ between kinds of records). +constexpr size_t FRAME_SIZE = 16u; + +// Helper functions for FRAME_SIZE alignment support. +size_t RoundUpToFrameSize(size_t size) { + return (size + FRAME_SIZE - 1) / FRAME_SIZE * FRAME_SIZE; +} + +// Internal structure of the record header. Must fit in FRAME_SIZE. +struct RecordHeader { + int64_t record_sequencing_id; + uint32_t record_size; // Size of the blob, not including RecordHeader + uint32_t record_hash; // Hash of the blob, not including RecordHeader + // Data starts right after the header. +}; +} // namespace + +// static +void StorageQueue::Create( + const QueueOptions& options, + StartUploadCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module, + base::OnceCallback<void(StatusOr<scoped_refptr<StorageQueue>>)> + completion_cb) { + // Initialize StorageQueue object loading the data. + class StorageQueueInitContext + : public TaskRunnerContext<StatusOr<scoped_refptr<StorageQueue>>> { + public: + StorageQueueInitContext( + scoped_refptr<StorageQueue> storage_queue, + base::OnceCallback<void(StatusOr<scoped_refptr<StorageQueue>>)> + callback) + : TaskRunnerContext<StatusOr<scoped_refptr<StorageQueue>>>( + std::move(callback), + storage_queue->sequenced_task_runner_), + storage_queue_(std::move(storage_queue)) { + DCHECK(storage_queue_); + } + + private: + // Context can only be deleted by calling Response method. + ~StorageQueueInitContext() override = default; + + void OnStart() override { + auto init_status = storage_queue_->Init(); + if (!init_status.ok()) { + Response(StatusOr<scoped_refptr<StorageQueue>>(init_status)); + return; + } + Response(std::move(storage_queue_)); + } + + scoped_refptr<StorageQueue> storage_queue_; + }; + + // Create StorageQueue object. + // Cannot use base::MakeRefCounted<StorageQueue>, because constructor is + // private. + scoped_refptr<StorageQueue> storage_queue = base::WrapRefCounted( + new StorageQueue(options, std::move(start_upload_cb), encryption_module)); + + // Asynchronously run initialization. + Start<StorageQueueInitContext>(std::move(storage_queue), + std::move(completion_cb)); +} + +StorageQueue::StorageQueue(const QueueOptions& options, + StartUploadCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module) + : options_(options), + start_upload_cb_(std::move(start_upload_cb)), + encryption_module_(encryption_module), + sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::BEST_EFFORT, base::MayBlock()})) { + DETACH_FROM_SEQUENCE(storage_queue_sequence_checker_); + DCHECK(write_contexts_queue_.empty()); +} + +StorageQueue::~StorageQueue() { + // TODO(b/153364303): Should be + // DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + + // Stop upload timer. + upload_timer_.AbandonAndStop(); + // Make sure no pending writes is present. + DCHECK(write_contexts_queue_.empty()); + + // Release all files. + ReleaseAllFileInstances(); +} + +Status StorageQueue::Init() { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + // Make sure the assigned directory exists. + base::File::Error error; + if (!base::CreateDirectoryAndGetError(options_.directory(), &error)) { + return Status( + error::UNAVAILABLE, + base::StrCat( + {"Storage queue directory '", options_.directory().MaybeAsASCII(), + "' does not exist, error=", base::File::ErrorToString(error)})); + } + // Enumerate data files and scan the last one to determine what sequence + // ids do we have (first and last). + base::flat_set<base::FilePath> used_files_set; + RETURN_IF_ERROR(EnumerateDataFiles(&used_files_set)); + RETURN_IF_ERROR(ScanLastFile()); + // In case of inavaliability default to a new generation id being a random + // number [1, max_int64] + generation_id_ = 1 + base::RandGenerator(std::numeric_limits<int64_t>::max()); + if (next_sequencing_id_ > 0) { + // Enumerate metadata files to determine what sequencing ids have + // last record digest. They might have metadata for sequencing ids + // beyond what data files had, because metadata is written ahead of the + // data, but must have metadata for the last data, because metadata is only + // removed once data is written. So we are picking the metadata matching the + // last sequencing id and load both digest and generation id from there. + const Status status = RestoreMetadata(&used_files_set); + // If there is no match, clear up everything we've found before and start + // a new generation from scratch. + // In the future we could possibly consider preserving the previous + // generation data, but will need to resolve multiple issues: + // 1) we would need to send the old generation before starting to send + // the new one, which could trigger a loss of data in the new generation. + // 2) we could end up with 3 or more generations, if the loss of metadata + // repeats. Which of them should be sent first (which one is expected + // by the server)? + // 3) different generations might include the same sequencing ids; + // how do we resolve file naming then? Should we add generation id + // to the file name too? + // Because of all this, for now we just drop the old generation data + // and start the new one from scratch. + if (!status.ok()) { + LOG(ERROR) << "Failed to restore metadata, status=" << status; + // Reset all parameters as they were at the beginning of Init(). + // Some of them might have been changed earlier. + next_sequencing_id_ = 0; + first_sequencing_id_ = 0; + first_unconfirmed_sequencing_id_ = base::nullopt; + last_record_digest_ = base::nullopt; + ReleaseAllFileInstances(); + used_files_set.clear(); + } + } + // Delete all files except used ones. + DeleteUnusedFiles(used_files_set); + // Initiate periodic uploading, if needed. + if (!options_.upload_period().is_zero()) { + upload_timer_.Start(FROM_HERE, options_.upload_period(), this, + &StorageQueue::Flush); + } + return Status::StatusOK(); +} + +void StorageQueue::UpdateRecordDigest(WrappedRecord* wrapped_record) { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + // Attach last record digest, if present. + if (last_record_digest_.has_value()) { + *wrapped_record->mutable_last_record_digest() = last_record_digest_.value(); + } + + // Calculate new record digest. + { + std::string serialized_record; + wrapped_record->record().SerializeToString(&serialized_record); + *wrapped_record->mutable_record_digest() = + crypto::SHA256HashString(serialized_record); + DCHECK_EQ(wrapped_record->record_digest().size(), crypto::kSHA256Length); + } + + // Store it in the record (for self-verification by the server). + last_record_digest_ = wrapped_record->record_digest(); +} + +StatusOr<int64_t> StorageQueue::AddDataFile( + const base::FilePath& full_name, + const base::FileEnumerator::FileInfo& file_info) { + const auto extension = full_name.Extension(); + if (extension.empty()) { + return Status(error::INTERNAL, + base::StrCat({"File has no extension: '", + full_name.MaybeAsASCII(), "'"})); + } + int64_t file_sequencing_id = 0; + bool success = base::StringToInt64(extension.substr(1), &file_sequencing_id); + if (!success) { + return Status(error::INTERNAL, + base::StrCat({"File extension does not parse: '", + full_name.MaybeAsASCII(), "'"})); + } + auto file_or_status = SingleFile::Create(full_name, file_info.GetSize()); + if (!file_or_status.ok()) { + return file_or_status.status(); + } + if (!files_.emplace(file_sequencing_id, file_or_status.ValueOrDie()).second) { + return Status(error::ALREADY_EXISTS, + base::StrCat({"Sequencing duplicated: '", + full_name.MaybeAsASCII(), "'"})); + } + return file_sequencing_id; +} + +Status StorageQueue::EnumerateDataFiles( + base::flat_set<base::FilePath>* used_files_set) { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + // We need to set first_sequencing_id_ to 0 if this is the initialization + // of an empty StorageQueue, and to the lowest sequencing id among all + // existing files, if it was already used. + base::Optional<int64_t> first_sequencing_id; + base::FileEnumerator dir_enum( + options_.directory(), + /*recursive=*/false, base::FileEnumerator::FILES, + base::StrCat({options_.file_prefix(), FILE_PATH_LITERAL(".*")})); + base::FilePath full_name; + while (full_name = dir_enum.Next(), !full_name.empty()) { + const auto file_sequencing_id_result = + AddDataFile(full_name, dir_enum.GetInfo()); + if (!file_sequencing_id_result.ok()) { + LOG(WARNING) << "Failed to add file " << full_name.MaybeAsASCII() + << ", status=" << file_sequencing_id_result.status(); + continue; + } + used_files_set->emplace(full_name); // File is in use. + if (!first_sequencing_id.has_value() || + first_sequencing_id.value() > file_sequencing_id_result.ValueOrDie()) { + first_sequencing_id = file_sequencing_id_result.ValueOrDie(); + } + } + // first_sequencing_id.has_value() is true only if we found some files. + // Otherwise it is false, the StorageQueue is being initialized for the + // first time, and we need to set first_sequencing_id_ to 0. + first_sequencing_id_ = + first_sequencing_id.has_value() ? first_sequencing_id.value() : 0; + return Status::StatusOK(); +} + +Status StorageQueue::ScanLastFile() { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + next_sequencing_id_ = 0; + if (files_.empty()) { + return Status::StatusOK(); + } + next_sequencing_id_ = files_.rbegin()->first; + // Scan the file. Open it and leave open, because it might soon be needed + // again (for the next or repeated Upload), and we won't waste time closing + // and reopening it. If the file remains open for too long, it will auto-close + // by timer. + scoped_refptr<SingleFile> last_file = files_.rbegin()->second.get(); + auto open_status = last_file->Open(/*read_only=*/false); + if (!open_status.ok()) { + LOG(ERROR) << "Error opening file " << last_file->name() + << ", status=" << open_status; + return Status(error::DATA_LOSS, base::StrCat({"Error opening file: '", + last_file->name(), "'"})); + } + const size_t max_buffer_size = + RoundUpToFrameSize(options_.max_record_size()) + + RoundUpToFrameSize(sizeof(RecordHeader)); + uint32_t pos = 0; + for (;;) { + // Read the header + auto read_result = + last_file->Read(pos, sizeof(RecordHeader), max_buffer_size); + if (read_result.status().error_code() == error::OUT_OF_RANGE) { + // End of file detected. + break; + } + if (!read_result.ok()) { + // Error detected. + LOG(ERROR) << "Error reading file " << last_file->name() + << ", status=" << read_result.status(); + break; + } + pos += read_result.ValueOrDie().size(); + if (read_result.ValueOrDie().size() < sizeof(RecordHeader)) { + // Error detected. + LOG(ERROR) << "Incomplete record header in file " << last_file->name(); + break; + } + // Copy the header, since the buffer might be overwritten later on. + const RecordHeader header = + *reinterpret_cast<const RecordHeader*>(read_result.ValueOrDie().data()); + // Read the data (rounded to frame size). + const size_t data_size = RoundUpToFrameSize(header.record_size); + read_result = last_file->Read(pos, data_size, max_buffer_size); + if (!read_result.ok()) { + // Error detected. + LOG(ERROR) << "Error reading file " << last_file->name() + << ", status=" << read_result.status(); + break; + } + pos += read_result.ValueOrDie().size(); + if (read_result.ValueOrDie().size() < data_size) { + // Error detected. + LOG(ERROR) << "Incomplete record in file " << last_file->name(); + break; + } + // Verify sequencing id. + if (header.record_sequencing_id != next_sequencing_id_) { + LOG(ERROR) << "sequencing id mismatch, expected=" << next_sequencing_id_ + << ", actual=" << header.record_sequencing_id << ", file " + << last_file->name(); + break; + } + // Verify record hash. + uint32_t actual_record_hash = base::PersistentHash( + read_result.ValueOrDie().data(), header.record_size); + if (header.record_hash != actual_record_hash) { + LOG(ERROR) << "Hash mismatch, seq=" << header.record_sequencing_id + << " actual_hash=" << std::hex << actual_record_hash + << " expected_hash=" << std::hex << header.record_hash; + break; + } + // Everything looks all right. Advance the sequencing id. + ++next_sequencing_id_; + } + return Status::StatusOK(); +} + +StatusOr<scoped_refptr<StorageQueue::SingleFile>> StorageQueue::AssignLastFile( + size_t size) { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + if (files_.empty()) { + // Create the very first file (empty). + ASSIGN_OR_RETURN( + scoped_refptr<SingleFile> file, + SingleFile::Create( + options_.directory() + .Append(options_.file_prefix()) + .AddExtensionASCII(base::NumberToString(next_sequencing_id_)), + /*size=*/0)); + next_sequencing_id_ = 0; + auto insert_result = files_.emplace(next_sequencing_id_, file); + DCHECK(insert_result.second); + } + if (size > options_.max_record_size()) { + return Status(error::OUT_OF_RANGE, "Too much data to be recorded at once"); + } + scoped_refptr<SingleFile> last_file = files_.rbegin()->second; + if (last_file->size() > 0 && // Cannot have a file with no records. + last_file->size() + size + sizeof(RecordHeader) + FRAME_SIZE > + options_.single_file_size()) { + // The last file will become too large, asynchronously close it and add + // new. + last_file->Close(); + ASSIGN_OR_RETURN(last_file, OpenNewWriteableFile()); + } + return last_file; +} + +StatusOr<scoped_refptr<StorageQueue::SingleFile>> +StorageQueue::OpenNewWriteableFile() { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + ASSIGN_OR_RETURN( + scoped_refptr<SingleFile> new_file, + SingleFile::Create( + options_.directory() + .Append(options_.file_prefix()) + .AddExtensionASCII(base::NumberToString(next_sequencing_id_)), + /*size=*/0)); + RETURN_IF_ERROR(new_file->Open(/*read_only=*/false)); + auto insert_result = files_.emplace(next_sequencing_id_, new_file); + if (!insert_result.second) { + return Status( + error::ALREADY_EXISTS, + base::StrCat({"Sequencing id already assigned: '", + base::NumberToString(next_sequencing_id_), "'"})); + } + return new_file; +} + +Status StorageQueue::WriteHeaderAndBlock( + base::StringPiece data, + scoped_refptr<StorageQueue::SingleFile> file) { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + // Prepare header. + RecordHeader header; + // Pad to the whole frame, if necessary. + const size_t total_size = RoundUpToFrameSize(sizeof(header) + data.size()); + // Assign sequencing id. + header.record_sequencing_id = next_sequencing_id_++; + header.record_hash = base::PersistentHash(data.data(), data.size()); + header.record_size = data.size(); + // Write to the last file, update sequencing id. + auto open_status = file->Open(/*read_only=*/false); + if (!open_status.ok()) { + return Status(error::ALREADY_EXISTS, + base::StrCat({"Cannot open file=", file->name(), + " status=", open_status.ToString()})); + } + if (!GetDiskResource()->Reserve(total_size)) { + return Status( + error::RESOURCE_EXHAUSTED, + base::StrCat({"Not enough disk space available to write into file=", + file->name()})); + } + auto write_status = file->Append(base::StringPiece( + reinterpret_cast<const char*>(&header), sizeof(header))); + if (!write_status.ok()) { + return Status(error::RESOURCE_EXHAUSTED, + base::StrCat({"Cannot write file=", file->name(), + " status=", write_status.status().ToString()})); + } + if (data.size() > 0) { + write_status = file->Append(data); + if (!write_status.ok()) { + return Status( + error::RESOURCE_EXHAUSTED, + base::StrCat({"Cannot write file=", file->name(), + " status=", write_status.status().ToString()})); + } + } + if (total_size > sizeof(header) + data.size()) { + // Fill in with random bytes. + const size_t pad_size = total_size - (sizeof(header) + data.size()); + char junk_bytes[FRAME_SIZE]; + crypto::RandBytes(junk_bytes, pad_size); + write_status = file->Append(base::StringPiece(&junk_bytes[0], pad_size)); + if (!write_status.ok()) { + return Status(error::RESOURCE_EXHAUSTED, + base::StrCat({"Cannot pad file=", file->name(), " status=", + write_status.status().ToString()})); + } + } + return Status::StatusOK(); +} + +Status StorageQueue::WriteMetadata() { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + // Synchronously write the metafile. + ASSIGN_OR_RETURN( + scoped_refptr<SingleFile> meta_file, + SingleFile::Create( + options_.directory() + .Append(METADATA_NAME) + .AddExtensionASCII(base::NumberToString(next_sequencing_id_)), + /*size=*/0)); + RETURN_IF_ERROR(meta_file->Open(/*read_only=*/false)); + // Account for the metadata file size. + DCHECK(last_record_digest_.has_value()); // Must be set by now. + if (!GetDiskResource()->Reserve(sizeof(generation_id_) + + last_record_digest_.value().size())) { + return Status( + error::RESOURCE_EXHAUSTED, + base::StrCat({"Not enough disk space available to write into file=", + meta_file->name()})); + } + // Write generation id. + auto append_result = meta_file->Append(base::StringPiece( + reinterpret_cast<const char*>(&generation_id_), sizeof(generation_id_))); + if (!append_result.ok()) { + return Status( + error::RESOURCE_EXHAUSTED, + base::StrCat({"Cannot write metafile=", meta_file->name(), + " status=", append_result.status().ToString()})); + } + // Write last record digest. + append_result = meta_file->Append(last_record_digest_.value()); + if (!append_result.ok()) { + return Status( + error::RESOURCE_EXHAUSTED, + base::StrCat({"Cannot write metafile=", meta_file->name(), + " status=", append_result.status().ToString()})); + } + if (append_result.ValueOrDie() != last_record_digest_.value().size()) { + return Status(error::DATA_LOSS, base::StrCat({"Failure writing metafile=", + meta_file->name()})); + } + meta_file->Close(); + // Switch the latest metafile. + meta_file_ = std::move(meta_file); + // Asynchronously delete all earlier metafiles. Do not wait for this to + // happen. + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT, base::MayBlock()}, + base::BindOnce(&StorageQueue::DeleteOutdatedMetadata, this, + next_sequencing_id_)); + return Status::StatusOK(); +} + +Status StorageQueue::RestoreMetadata( + base::flat_set<base::FilePath>* used_files_set) { + // Enumerate all meta-files into a map sequencing_id->file_path. + std::map<int64_t, std::pair<base::FilePath, size_t>> meta_files; + base::FileEnumerator dir_enum( + options_.directory(), + /*recursive=*/false, base::FileEnumerator::FILES, + base::StrCat({METADATA_NAME, FILE_PATH_LITERAL(".*")})); + base::FilePath full_name; + while (full_name = dir_enum.Next(), !full_name.empty()) { + const auto extension = dir_enum.GetInfo().GetName().Extension(); + if (extension.empty()) { + continue; + } + int64_t sequencing_id = 0; + bool success = base::StringToInt64( + dir_enum.GetInfo().GetName().Extension().substr(1), &sequencing_id); + if (!success) { + continue; + } + // Record file name and size. Ignore the result. + meta_files.emplace(sequencing_id, + std::make_pair(full_name, dir_enum.GetInfo().GetSize())); + } + // See whether we have a match for next_sequencing_id_ - 1. + DCHECK_GT(next_sequencing_id_, 0u); + auto it = meta_files.find(next_sequencing_id_ - 1); + if (it == meta_files.end()) { + // For now we fail in this case. Later on we will provide a generation + // switch. + return Status( + error::DATA_LOSS, + base::StrCat({"Cannot recover last record digest at ", + base::NumberToString(next_sequencing_id_ - 1)})); + } + // Match found. Load the metadata. + const base::FilePath meta_file_path = it->second.first; + ASSIGN_OR_RETURN(scoped_refptr<SingleFile> meta_file, + SingleFile::Create(meta_file_path, + /*size=*/it->second.second)); + RETURN_IF_ERROR(meta_file->Open(/*read_only=*/true)); + // Read generation id. + constexpr size_t max_buffer_size = + sizeof(generation_id_) + crypto::kSHA256Length; + auto read_result = + meta_file->Read(/*pos=*/0, sizeof(generation_id_), max_buffer_size); + if (!read_result.ok() || + read_result.ValueOrDie().size() != sizeof(generation_id_)) { + return Status(error::DATA_LOSS, + base::StrCat({"Cannot read metafile=", meta_file->name(), + " status=", read_result.status().ToString()})); + } + const int64_t generation_id = + *reinterpret_cast<const int64_t*>(read_result.ValueOrDie().data()); + // Read last record digest. + read_result = meta_file->Read(/*pos=*/sizeof(generation_id_), + crypto::kSHA256Length, max_buffer_size); + if (!read_result.ok() || + read_result.ValueOrDie().size() != crypto::kSHA256Length) { + return Status(error::DATA_LOSS, + base::StrCat({"Cannot read metafile=", meta_file->name(), + " status=", read_result.status().ToString()})); + } + // Everything read successfully, set the queue up. + generation_id_ = generation_id; + last_record_digest_ = std::string(read_result.ValueOrDie()); + meta_file_ = std::move(meta_file); + // Store used metadata file. + used_files_set->emplace(meta_file_path); + return Status::StatusOK(); +} + +void StorageQueue::DeleteUnusedFiles( + const base::flat_set<base::FilePath>& used_files_setused_files_set) { + // Note, that these files were not reserved against disk allowance and do not + // need to be discarded. + base::FileEnumerator dir_enum(options_.directory(), + /*recursive=*/true, + base::FileEnumerator::FILES); + base::FilePath full_name; + while (full_name = dir_enum.Next(), !full_name.empty()) { + if (used_files_setused_files_set.count(full_name) > 0) { + continue; // File is used, keep it. + } + base::DeleteFile(full_name); + } +} + +void StorageQueue::DeleteOutdatedMetadata(int64_t sequencing_id_to_keep) { + std::vector<std::pair<base::FilePath, uint64_t>> files_to_delete; + base::FileEnumerator dir_enum( + options_.directory(), + /*recursive=*/false, base::FileEnumerator::FILES, + base::StrCat({METADATA_NAME, FILE_PATH_LITERAL(".*")})); + base::FilePath full_name; + while (full_name = dir_enum.Next(), !full_name.empty()) { + const auto extension = dir_enum.GetInfo().GetName().Extension(); + if (extension.empty()) { + continue; + } + int64_t sequencing_id = 0; + bool success = base::StringToInt64( + dir_enum.GetInfo().GetName().Extension().substr(1), &sequencing_id); + if (!success) { + continue; + } + if (sequencing_id >= sequencing_id_to_keep) { + continue; + } + files_to_delete.emplace_back( + std::make_pair(full_name, dir_enum.GetInfo().GetSize())); + } + for (const auto& file_to_delete : files_to_delete) { + // Delete file on disk. Note: disk space has already been released when the + // metafile was destructed, and so we don't need to do that here. + base::DeleteFile(file_to_delete.first); // ignore result + } +} + +// Context for uploading data from the queue in proper sequence. +// Runs on a storage_queue->sequenced_task_runner_ +// Makes necessary calls to the provided |UploaderInterface|: +// repeatedly to ProcessRecord/ProcessGap, and Completed at the end. +// Sets references to potentially used files aside, and increments +// active_read_operations_ to make sure confirmation will not trigger +// files deletion. Decrements it upon completion (when this counter +// is zero, RemoveConfirmedData can delete the unused files). +class StorageQueue::ReadContext : public TaskRunnerContext<Status> { + public: + ReadContext(std::unique_ptr<UploaderInterface> uploader, + scoped_refptr<StorageQueue> storage_queue) + : TaskRunnerContext<Status>( + base::BindOnce(&UploaderInterface::Completed, + base::Unretained(uploader.get())), + storage_queue->sequenced_task_runner_), + uploader_(std::move(uploader)), + storage_queue_weakptr_factory_{storage_queue.get()} { + DCHECK(storage_queue.get()); + DCHECK(uploader_.get()); + DETACH_FROM_SEQUENCE(read_sequence_checker_); + } + + private: + // Context can only be deleted by calling Response method. + ~ReadContext() override = default; + + void OnStart() override { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + base::WeakPtr<StorageQueue> storage_queue = + storage_queue_weakptr_factory_.GetWeakPtr(); + if (!storage_queue) { + Response(Status(error::UNAVAILABLE, "StorageQueue shut down")); + return; + } + + // Fill in initial sequencing information to track progress: + // use minimum of first_sequencing_id_ and first_unconfirmed_sequencing_id_ + // if the latter has been recorded. + sequencing_info_.set_generation_id(storage_queue->generation_id_); + if (storage_queue->first_unconfirmed_sequencing_id_.has_value()) { + sequencing_info_.set_sequencing_id( + std::min(storage_queue->first_unconfirmed_sequencing_id_.value(), + storage_queue->first_sequencing_id_)); + } else { + sequencing_info_.set_sequencing_id(storage_queue->first_sequencing_id_); + } + + // If the last file is not empty (has at least one record), + // close it and create the new one, so that its records are + // also included in the reading. + const Status last_status = storage_queue->SwitchLastFileIfNotEmpty(); + if (!last_status.ok()) { + Response(last_status); + return; + } + + // Collect and set aside the files in the set that might have data + // for the Upload. + files_ = + storage_queue->CollectFilesForUpload(sequencing_info_.sequencing_id()); + if (files_.empty()) { + Response(Status(error::OUT_OF_RANGE, + "Sequencing id not found in StorageQueue.")); + return; + } + + // Register with storage_queue, to make sure selected files are not removed. + ++(storage_queue->active_read_operations_); + + // The first <seq.file> pair is the current file now, and we are at its + // start or ahead of it. + current_file_ = files_.begin(); + current_pos_ = 0; + + // If the first record we need to upload is unavailable, produce Gap record + // instead. + if (sequencing_info_.sequencing_id() < current_file_->first) { + CallGapUpload(/*count=*/current_file_->first - + sequencing_info_.sequencing_id()); + // Resume at ScheduleNextRecord. + return; + } + + StartUploading(storage_queue); + } + + void StartUploading(base::WeakPtr<StorageQueue> storage_queue) { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + // Read from it until the specified sequencing id is found. + for (int64_t sequencing_id = current_file_->first; + sequencing_id < sequencing_info_.sequencing_id(); ++sequencing_id) { + auto blob = EnsureBlob(storage_queue, sequencing_id); + if (blob.status().error_code() == error::OUT_OF_RANGE) { + // Reached end of file, switch to the next one (if present). + ++current_file_; + if (current_file_ == files_.end()) { + Response(Status::StatusOK()); + return; + } + current_pos_ = 0; + blob = EnsureBlob(storage_queue, sequencing_info_.sequencing_id()); + } + if (!blob.ok()) { + // File found to be corrupt. Produce Gap record till the start of next + // file, if present. + ++current_file_; + current_pos_ = 0; + uint64_t count = static_cast<uint64_t>( + (current_file_ == files_.end()) + ? 1 + : current_file_->first - sequencing_info_.sequencing_id()); + CallGapUpload(count); + // Resume at ScheduleNextRecord. + return; + } + } + + // Read and upload sequencing_info_.sequencing_id(). + CallRecordOrGap(storage_queue, sequencing_info_.sequencing_id()); + // Resume at ScheduleNextRecord. + } + + void OnCompletion() override { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + // Unregister with storage_queue. + if (!files_.empty()) { + base::WeakPtr<StorageQueue> storage_queue = + storage_queue_weakptr_factory_.GetWeakPtr(); + if (storage_queue) { + const auto count = --(storage_queue->active_read_operations_); + DCHECK_GE(count, 0); + } + } + } + + // Prepares the |blob| for uploading. + void CallCurrentRecord(base::StringPiece blob) { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + google::protobuf::io::ArrayInputStream blob_stream( // Zero-copy stream. + blob.data(), blob.size()); + EncryptedRecord encrypted_record; + if (!encrypted_record.ParseFromZeroCopyStream(&blob_stream)) { + LOG(ERROR) << "Failed to parse record, seq=" + << sequencing_info_.sequencing_id(); + CallGapUpload(/*count=*/1); + // Resume at ScheduleNextRecord. + return; + } + CallRecordUpload(std::move(encrypted_record)); + } + + // Completes sequencing information and makes a call to UploaderInterface + // instance provided by user, which can place processing of the record on any + // thread(s). Once it returns, it will schedule NextRecord to execute on the + // sequential thread runner of this StorageQueue. If |encrypted_record| is + // empty (has no |encrypted_wrapped_record| and/or |encryption_info|), it + // indicates a gap notification. + void CallRecordUpload(EncryptedRecord encrypted_record) { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + if (encrypted_record.has_sequencing_information()) { + LOG(ERROR) << "Sequencing information already present, seq=" + << sequencing_info_.sequencing_id(); + CallGapUpload(/*count=*/1); + // Resume at ScheduleNextRecord. + return; + } + // Fill in sequencing information. + // Priority is attached by the Storage layer. + *encrypted_record.mutable_sequencing_information() = sequencing_info_; + uploader_->ProcessRecord(std::move(encrypted_record), + base::BindOnce(&ReadContext::ScheduleNextRecord, + base::Unretained(this))); + // Move sequencing forward (ScheduleNextRecord will see this). + sequencing_info_.set_sequencing_id(sequencing_info_.sequencing_id() + 1); + } + + void CallGapUpload(uint64_t count) { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + if (count == 0u) { + // No records skipped. + NextRecord(/*more_records=*/true); + return; + } + uploader_->ProcessGap(sequencing_info_, count, + base::BindOnce(&ReadContext::ScheduleNextRecord, + base::Unretained(this))); + // Move sequencing forward (ScheduleNextRecord will see this). + sequencing_info_.set_sequencing_id(sequencing_info_.sequencing_id() + + count); + } + + // Schedules NextRecord to execute on the StorageQueue sequential task runner. + void ScheduleNextRecord(bool more_records) { + Schedule(&ReadContext::NextRecord, base::Unretained(this), more_records); + } + + // If more records are expected, retrieves the next record (if present) and + // sends for processing, or calls Response with error status. Otherwise, call + // Response(OK). + void NextRecord(bool more_records) { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + if (!more_records) { + Response(Status::StatusOK()); // Requested to stop reading. + return; + } + base::WeakPtr<StorageQueue> storage_queue = + storage_queue_weakptr_factory_.GetWeakPtr(); + if (!storage_queue) { + Response(Status(error::UNAVAILABLE, "StorageQueue shut down")); + return; + } + // If reached end of the last file, finish reading. + if (current_file_ == files_.end()) { + Response(Status::StatusOK()); + return; + } + // sequencing_info_.sequencing_id() blob is ready. + CallRecordOrGap(storage_queue, sequencing_info_.sequencing_id()); + // Resume at ScheduleNextRecord. + } + + // Loads blob from the current file - reads header first, and then the body. + // (SingleFile::Read call makes sure all the data is in the buffer). + // After reading, verifies that data matches the hash stored in the header. + // If everything checks out, returns the reference to the data in the buffer: + // the buffer remains intact until the next call to SingleFile::Read. + // If anything goes wrong (file is shorter than expected, or record hash does + // not match), returns error. + StatusOr<base::StringPiece> EnsureBlob( + base::WeakPtr<StorageQueue> storage_queue, + int64_t sequencing_id) { + DCHECK_CALLED_ON_VALID_SEQUENCE(read_sequence_checker_); + + // Test only: simulate error, if requested. + if (storage_queue->test_injected_fail_sequencing_ids_.count(sequencing_id) > + 0) { + return Status(error::INTERNAL, + base::StrCat({"Simulated failure, seq=", + base::NumberToString(sequencing_id)})); + } + + // Read from the current file at the current offset. + RETURN_IF_ERROR(current_file_->second->Open(/*read_only=*/true)); + const size_t max_buffer_size = + RoundUpToFrameSize(storage_queue->options_.max_record_size()) + + RoundUpToFrameSize(sizeof(RecordHeader)); + auto read_result = current_file_->second->Read( + current_pos_, sizeof(RecordHeader), max_buffer_size); + RETURN_IF_ERROR(read_result.status()); + auto header_data = read_result.ValueOrDie(); + if (header_data.empty()) { + // No more blobs. + return Status(error::OUT_OF_RANGE, "Reached end of data"); + } + current_pos_ += header_data.size(); + if (header_data.size() != sizeof(RecordHeader)) { + // File corrupt, header incomplete. + return Status( + error::INTERNAL, + base::StrCat({"File corrupt: ", current_file_->second->name()})); + } + // Copy the header out (its memory can be overwritten when reading rest of + // the data). + const RecordHeader header = + *reinterpret_cast<const RecordHeader*>(header_data.data()); + if (header.record_sequencing_id != sequencing_id) { + return Status( + error::INTERNAL, + base::StrCat( + {"File corrupt: ", current_file_->second->name(), + " seq=", base::NumberToString(header.record_sequencing_id), + " expected=", base::NumberToString(sequencing_id)})); + } + // Read the record blob (align size to FRAME_SIZE). + const size_t data_size = RoundUpToFrameSize(header.record_size); + // From this point on, header in memory is no longer used and can be + // overwritten when reading rest of the data. + read_result = + current_file_->second->Read(current_pos_, data_size, max_buffer_size); + RETURN_IF_ERROR(read_result.status()); + current_pos_ += read_result.ValueOrDie().size(); + if (read_result.ValueOrDie().size() != data_size) { + // File corrupt, blob incomplete. + return Status( + error::INTERNAL, + base::StrCat( + {"File corrupt: ", current_file_->second->name(), + " size=", base::NumberToString(read_result.ValueOrDie().size()), + " expected=", base::NumberToString(data_size)})); + } + // Verify record hash. + uint32_t actual_record_hash = base::PersistentHash( + read_result.ValueOrDie().data(), header.record_size); + if (header.record_hash != actual_record_hash) { + return Status( + error::INTERNAL, + base::StrCat( + {"File corrupt: ", current_file_->second->name(), " seq=", + base::NumberToString(header.record_sequencing_id), " hash=", + base::HexEncode( + reinterpret_cast<const uint8_t*>(&header.record_hash), + sizeof(header.record_hash)), + " expected=", + base::HexEncode( + reinterpret_cast<const uint8_t*>(&actual_record_hash), + sizeof(actual_record_hash))})); + } + return read_result.ValueOrDie().substr(0, header.record_size); + } + + void CallRecordOrGap(base::WeakPtr<StorageQueue> storage_queue, + int64_t sequencing_id) { + auto blob = EnsureBlob(storage_queue, sequencing_info_.sequencing_id()); + if (blob.status().error_code() == error::OUT_OF_RANGE) { + // Reached end of file, switch to the next one (if present). + ++current_file_; + if (current_file_ == files_.end()) { + Response(Status::StatusOK()); + return; + } + current_pos_ = 0; + blob = EnsureBlob(storage_queue, sequencing_info_.sequencing_id()); + } + if (!blob.ok()) { + // File found to be corrupt. Produce Gap record till the start of next + // file, if present. + ++current_file_; + current_pos_ = 0; + uint64_t count = static_cast<uint64_t>( + (current_file_ == files_.end()) + ? 1 + : current_file_->first - sequencing_info_.sequencing_id()); + CallGapUpload(count); + // Resume at ScheduleNextRecord. + return; + } + CallCurrentRecord(blob.ValueOrDie()); + // Resume at ScheduleNextRecord. + } + + // Files that will be read (in order of sequencing ids). + std::map<int64_t, scoped_refptr<SingleFile>> files_; + SequencingInformation sequencing_info_; + uint32_t current_pos_; + std::map<int64_t, scoped_refptr<SingleFile>>::iterator current_file_; + const std::unique_ptr<UploaderInterface> uploader_; + base::WeakPtrFactory<StorageQueue> storage_queue_weakptr_factory_; + + SEQUENCE_CHECKER(read_sequence_checker_); +}; + +class StorageQueue::WriteContext : public TaskRunnerContext<Status> { + public: + WriteContext(Record record, + base::OnceCallback<void(Status)> write_callback, + scoped_refptr<StorageQueue> storage_queue) + : TaskRunnerContext<Status>(std::move(write_callback), + storage_queue->sequenced_task_runner_), + storage_queue_(storage_queue), + record_(std::move(record)), + in_contexts_queue_(storage_queue->write_contexts_queue_.end()) { + DCHECK(storage_queue.get()); + DETACH_FROM_SEQUENCE(write_sequence_checker_); + } + + private: + // Context can only be deleted by calling Response method. + ~WriteContext() override { + DCHECK_CALLED_ON_VALID_SEQUENCE(write_sequence_checker_); + + // If still in queue, remove it (something went wrong). + if (in_contexts_queue_ != storage_queue_->write_contexts_queue_.end()) { + DCHECK_EQ(storage_queue_->write_contexts_queue_.front(), this); + storage_queue_->write_contexts_queue_.erase(in_contexts_queue_); + } + + // If there is the context at the front of the queue and its buffer is + // filled in, schedule respective |Write| to happen now. + if (!storage_queue_->write_contexts_queue_.empty() && + !storage_queue_->write_contexts_queue_.front()->buffer_.empty()) { + storage_queue_->write_contexts_queue_.front()->Schedule( + &WriteContext::ResumeWriteRecord, + base::Unretained(storage_queue_->write_contexts_queue_.front())); + } + + // If no uploader is needed, we are done. + if (!uploader_) { + return; + } + + // Otherwise initiate Upload right after writing + // finished and respond back when reading Upload is done. + // Note: new uploader created synchronously before scheduling Upload. + Start<ReadContext>(std::move(uploader_), storage_queue_); + } + + void OnStart() override { + DCHECK_CALLED_ON_VALID_SEQUENCE(write_sequence_checker_); + + // Make sure the record is valid. + if (!record_.has_destination()) { + Response(Status(error::FAILED_PRECONDITION, + "Malformed record: missing destination")); + return; + } + if (!record_.has_dm_token()) { + Response(Status(error::FAILED_PRECONDITION, + "Malformed record: missing dm_token")); + return; + } + + // Wrap the record. + WrappedRecord wrapped_record; + *wrapped_record.mutable_record() = std::move(record_); + + // Calculate and attach record digest. + storage_queue_->UpdateRecordDigest(&wrapped_record); + + // Add context to the end of the queue. + in_contexts_queue_ = storage_queue_->write_contexts_queue_.insert( + storage_queue_->write_contexts_queue_.end(), this); + + // Serialize and encrypt wrapped record on a thread pool. + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce(&WriteContext::SerializeAndEncryptWrappedRecord, + base::Unretained(this), std::move(wrapped_record))); + } + + void SerializeAndEncryptWrappedRecord(WrappedRecord wrapped_record) { + // Serialize wrapped record into a string. + ScopedReservation scoped_reservation(wrapped_record.ByteSizeLong(), + GetMemoryResource()); + if (!scoped_reservation.reserved()) { + Schedule(&ReadContext::Response, base::Unretained(this), + Status(error::RESOURCE_EXHAUSTED, + "Not enough memory for the write buffer")); + return; + } + + std::string buffer; + if (!wrapped_record.SerializeToString(&buffer)) { + Schedule(&ReadContext::Response, base::Unretained(this), + Status(error::DATA_LOSS, "Cannot serialize record")); + return; + } + // Release wrapped record memory, so scoped reservation may act. + wrapped_record.Clear(); + + // Encrypt the result. + storage_queue_->encryption_module_->EncryptRecord( + buffer, base::BindOnce(&WriteContext::OnEncryptedRecordReady, + base::Unretained(this))); + } + + void OnEncryptedRecordReady( + StatusOr<EncryptedRecord> encrypted_record_result) { + if (!encrypted_record_result.ok()) { + // Failed to serialize or encrypt. + Schedule(&ReadContext::Response, base::Unretained(this), + encrypted_record_result.status()); + return; + } + + // Serialize encrypted record. + ScopedReservation scoped_reservation( + encrypted_record_result.ValueOrDie().ByteSizeLong(), + GetMemoryResource()); + if (!scoped_reservation.reserved()) { + Schedule(&ReadContext::Response, base::Unretained(this), + Status(error::RESOURCE_EXHAUSTED, + "Not enough memory for the write buffer")); + return; + } + std::string buffer; + if (!encrypted_record_result.ValueOrDie().SerializeToString(&buffer)) { + Schedule(&ReadContext::Response, base::Unretained(this), + Status(error::DATA_LOSS, "Cannot serialize EncryptedRecord")); + return; + } + // Release encrypted record memory, so scoped reservation may act. + encrypted_record_result.ValueOrDie().Clear(); + + // Write into storage on sequntial task runner. + Schedule(&WriteContext::WriteRecord, base::Unretained(this), + std::move(buffer)); + } + + void WriteRecord(std::string buffer) { + DCHECK_CALLED_ON_VALID_SEQUENCE(write_sequence_checker_); + buffer_.swap(buffer); + + ResumeWriteRecord(); + } + + void ResumeWriteRecord() { + DCHECK_CALLED_ON_VALID_SEQUENCE(write_sequence_checker_); + + // If we are not at the head of the queue, delay write and expect to be + // reactivated later. + DCHECK(in_contexts_queue_ != storage_queue_->write_contexts_queue_.end()); + if (storage_queue_->write_contexts_queue_.front() != this) { + return; + } + + // We are at the head of the queue, remove ourselves. + storage_queue_->write_contexts_queue_.pop_front(); + in_contexts_queue_ = storage_queue_->write_contexts_queue_.end(); + + // Prepare uploader, if need to run it after Write. + if (storage_queue_->options_.upload_period().is_zero()) { + StatusOr<std::unique_ptr<UploaderInterface>> uploader = + storage_queue_->start_upload_cb_.Run(); + if (uploader.ok()) { + uploader_ = std::move(uploader.ValueOrDie()); + } else { + LOG(ERROR) << "Failed to provide the Uploader, status=" + << uploader.status(); + } + } + + DCHECK(!buffer_.empty()); + StatusOr<scoped_refptr<SingleFile>> assign_result = + storage_queue_->AssignLastFile(buffer_.size()); + if (!assign_result.ok()) { + Response(assign_result.status()); + return; + } + scoped_refptr<SingleFile> last_file = assign_result.ValueOrDie(); + + // Writing metadata ahead of the data write. + Status write_result = storage_queue_->WriteMetadata(); + if (!write_result.ok()) { + Response(write_result); + return; + } + + // Write header and block. + write_result = + storage_queue_->WriteHeaderAndBlock(buffer_, std::move(last_file)); + if (!write_result.ok()) { + Response(write_result); + return; + } + + Response(Status::StatusOK()); + } + + scoped_refptr<StorageQueue> storage_queue_; + + Record record_; + + // Position in the |storage_queue_|->|write_contexts_queue_|. + // We use it in order to detect whether the context is in the queue + // and to remove it from the queue, when the time comes. + std::list<WriteContext*>::iterator in_contexts_queue_; + + // Write buffer. When filled in (after encryption), |WriteRecord| can be + // executed. Empty until encryption is done. + std::string buffer_; + + // Upload provider (if any). + std::unique_ptr<UploaderInterface> uploader_; + + SEQUENCE_CHECKER(write_sequence_checker_); +}; + +void StorageQueue::Write(Record record, + base::OnceCallback<void(Status)> completion_cb) { + Start<WriteContext>(std::move(record), std::move(completion_cb), this); +} + +Status StorageQueue::SwitchLastFileIfNotEmpty() { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + if (files_.empty()) { + return Status(error::OUT_OF_RANGE, + "No files in the queue"); // No files in this queue yet. + } + if (files_.rbegin()->second->size() == 0) { + return Status::StatusOK(); // Already empty. + } + files_.rbegin()->second->Close(); + ASSIGN_OR_RETURN(scoped_refptr<SingleFile> last_file, OpenNewWriteableFile()); + return Status::StatusOK(); +} + +std::map<int64_t, scoped_refptr<StorageQueue::SingleFile>> +StorageQueue::CollectFilesForUpload(int64_t sequencing_id) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + // Locate the first file based on sequencing id. + auto file_it = files_.find(sequencing_id); + if (file_it == files_.end()) { + file_it = files_.upper_bound(sequencing_id); + if (file_it != files_.begin()) { + --file_it; + } + } + + // Create references to the files that will be uploaded. + // Exclude the last file (still being written). + std::map<int64_t, scoped_refptr<SingleFile>> files; + for (; file_it != files_.end() && + file_it->second.get() != files_.rbegin()->second.get(); + ++file_it) { + files.emplace(file_it->first, file_it->second); // Adding reference. + } + return files; +} + +class StorageQueue::ConfirmContext : public TaskRunnerContext<Status> { + public: + ConfirmContext(base::Optional<int64_t> sequencing_id, + bool force, + base::OnceCallback<void(Status)> end_callback, + scoped_refptr<StorageQueue> storage_queue) + : TaskRunnerContext<Status>(std::move(end_callback), + storage_queue->sequenced_task_runner_), + sequencing_id_(sequencing_id), + force_(force), + storage_queue_(storage_queue) { + DCHECK(storage_queue.get()); + DETACH_FROM_SEQUENCE(confirm_sequence_checker_); + } + + private: + // Context can only be deleted by calling Response method. + ~ConfirmContext() override = default; + + void OnStart() override { + DCHECK_CALLED_ON_VALID_SEQUENCE(confirm_sequence_checker_); + if (force_) { + storage_queue_->first_unconfirmed_sequencing_id_ = + sequencing_id_.has_value() ? (sequencing_id_.value() + 1) : 0; + Response(Status::StatusOK()); + } else { + Response(sequencing_id_.has_value() + ? storage_queue_->RemoveConfirmedData(sequencing_id_.value()) + : Status::StatusOK()); + } + } + + // Confirmed sequencing id. + base::Optional<int64_t> sequencing_id_; + + bool force_; + + scoped_refptr<StorageQueue> storage_queue_; + + SEQUENCE_CHECKER(confirm_sequence_checker_); +}; + +void StorageQueue::Confirm(base::Optional<int64_t> sequencing_id, + bool force, + base::OnceCallback<void(Status)> completion_cb) { + Start<ConfirmContext>(sequencing_id, force, std::move(completion_cb), this); +} + +Status StorageQueue::RemoveConfirmedData(int64_t sequencing_id) { + DCHECK_CALLED_ON_VALID_SEQUENCE(storage_queue_sequence_checker_); + // Update first unconfirmed id, unless new one is lower. + if (!first_unconfirmed_sequencing_id_.has_value() || + first_unconfirmed_sequencing_id_.value() <= sequencing_id) { + first_unconfirmed_sequencing_id_ = sequencing_id + 1; + } + // Update first available id, if new one is higher. + if (first_sequencing_id_ <= sequencing_id) { + first_sequencing_id_ = sequencing_id + 1; + } + if (active_read_operations_ > 0) { + // If there are read locks registered, bail out + // (expect to remove unused files later). + return Status::StatusOK(); + } + // Remove all files with sequencing ids below or equal only. + // Note: files_ cannot be empty ever (there is always the current + // file for writing). + for (;;) { + DCHECK(!files_.empty()) << "Empty storage queue"; + auto next_it = files_.begin(); + ++next_it; // Need to consider the next file. + if (next_it == files_.end()) { + // We are on the last file, keep it. + break; + } + if (next_it->first > sequencing_id + 1) { + // Current file ends with (next_it->first - 1). + // If it is sequencing_id >= (next_it->first - 1), we must keep it. + break; + } + // Current file holds only ids <= sequencing_id. + // Delete it. + files_.begin()->second->Close(); + if (files_.begin()->second->Delete().ok()) { + files_.erase(files_.begin()); + } + } + // Even if there were errors, ignore them. + return Status::StatusOK(); +} + +void StorageQueue::Flush() { + // Note: new uploader created every time Flush is called. + StatusOr<std::unique_ptr<UploaderInterface>> uploader = + start_upload_cb_.Run(); + if (!uploader.ok()) { + LOG(ERROR) << "Failed to provide the Uploader, status=" + << uploader.status(); + return; + } + Start<ReadContext>(std::move(uploader.ValueOrDie()), this); +} + +void StorageQueue::ReleaseAllFileInstances() { + files_.clear(); + meta_file_.reset(); +} + +void StorageQueue::TestInjectBlockReadErrors( + std::initializer_list<int64_t> sequencing_ids) { + test_injected_fail_sequencing_ids_ = sequencing_ids; +} + +// +// SingleFile implementation +// +StatusOr<scoped_refptr<StorageQueue::SingleFile>> +StorageQueue::SingleFile::Create(const base::FilePath& filename, int64_t size) { + if (!GetDiskResource()->Reserve(size)) { + LOG(WARNING) << "Disk space exceeded adding file " + << filename.MaybeAsASCII(); + return Status( + error::RESOURCE_EXHAUSTED, + base::StrCat({"Not enough disk space available to include file=", + filename.MaybeAsASCII()})); + } + // Cannot use base::MakeRefCounted, since the constructor is private. + return scoped_refptr<StorageQueue::SingleFile>( + new SingleFile(filename, size)); +} + +StorageQueue::SingleFile::SingleFile(const base::FilePath& filename, + int64_t size) + : filename_(filename), size_(size) {} + +StorageQueue::SingleFile::~SingleFile() { + GetDiskResource()->Discard(size_); + Close(); + handle_.reset(); +} + +Status StorageQueue::SingleFile::Open(bool read_only) { + if (handle_) { + DCHECK_EQ(is_readonly(), read_only); + // TODO(b/157943192): Restart auto-closing timer. + return Status::StatusOK(); + } + handle_ = std::make_unique<base::File>( + filename_, read_only ? (base::File::FLAG_OPEN | base::File::FLAG_READ) + : (base::File::FLAG_OPEN_ALWAYS | + base::File::FLAG_APPEND | base::File::FLAG_READ)); + if (!handle_ || !handle_->IsValid()) { + return Status(error::DATA_LOSS, + base::StrCat({"Cannot open file=", name(), " for ", + read_only ? "read" : "append"})); + } + is_readonly_ = read_only; + if (!read_only) { + int64_t file_size = handle_->GetLength(); + if (file_size < 0) { + return Status(error::DATA_LOSS, + base::StrCat({"Cannot get size of file=", name()})); + } + size_ = static_cast<uint64_t>(file_size); + } + return Status::StatusOK(); +} + +void StorageQueue::SingleFile::Close() { + if (!handle_) { + // TODO(b/157943192): Restart auto-closing timer. + return; + } + handle_.reset(); + is_readonly_ = base::nullopt; + if (buffer_) { + buffer_.reset(); + GetMemoryResource()->Discard(buffer_size_); + } +} + +Status StorageQueue::SingleFile::Delete() { + DCHECK(!handle_); + GetDiskResource()->Discard(size_); + size_ = 0; + if (!base::DeleteFile(filename_)) { + return Status(error::DATA_LOSS, + base::StrCat({"Cannot delete file=", name()})); + } + return Status::StatusOK(); +} + +StatusOr<base::StringPiece> StorageQueue::SingleFile::Read( + uint32_t pos, + uint32_t size, + size_t max_buffer_size) { + if (!handle_) { + return Status(error::UNAVAILABLE, base::StrCat({"File not open ", name()})); + } + if (size > max_buffer_size) { + return Status(error::RESOURCE_EXHAUSTED, "Too much data to read"); + } + if (size_ == 0) { + // Empty file, return EOF right away. + return Status(error::OUT_OF_RANGE, "End of file"); + } + buffer_size_ = std::min(max_buffer_size, RoundUpToFrameSize(size_)); + // If no buffer yet, allocate. + // TODO(b/157943192): Add buffer management - consider adding an UMA for + // tracking the average + peak memory the Storage module is consuming. + if (!buffer_) { + // Register with resource management. + if (!GetMemoryResource()->Reserve(buffer_size_)) { + return Status(error::RESOURCE_EXHAUSTED, + "Not enough memory for the read buffer"); + } + buffer_ = std::make_unique<char[]>(buffer_size_); + data_start_ = data_end_ = 0; + file_position_ = 0; + } + // If file position does not match, reset buffer. + if (pos != file_position_) { + data_start_ = data_end_ = 0; + file_position_ = pos; + } + // If expected data size does not fit into the buffer, move what's left to the + // start. + if (data_start_ + size > buffer_size_) { + DCHECK_GT(data_start_, 0u); // Cannot happen if 0. + memmove(buffer_.get(), buffer_.get() + data_start_, + data_end_ - data_start_); + data_end_ -= data_start_; + data_start_ = 0; + } + size_t actual_size = data_end_ - data_start_; + pos += actual_size; + while (actual_size < size) { + // Read as much as possible. + DCHECK_LT(data_end_, buffer_size_); + const int32_t result = + handle_->Read(pos, reinterpret_cast<char*>(buffer_.get() + data_end_), + buffer_size_ - data_end_); + if (result < 0) { + return Status( + error::DATA_LOSS, + base::StrCat({"File read error=", + handle_->ErrorToString(handle_->GetLastFileError()), + " ", name()})); + } + if (result == 0) { + break; + } + pos += result; + data_end_ += result; + DCHECK_LE(data_end_, buffer_size_); + actual_size += result; + } + if (actual_size > size) { + actual_size = size; + } + // If nothing read, report end of file. + if (actual_size == 0) { + return Status(error::OUT_OF_RANGE, "End of file"); + } + // Prepare reference to actually loaded data. + auto read_data = base::StringPiece(buffer_.get() + data_start_, actual_size); + // Move start and file position to after that data. + data_start_ += actual_size; + file_position_ += actual_size; + DCHECK_LE(data_start_, data_end_); + // Return what has been loaded. + return read_data; +} + +StatusOr<uint32_t> StorageQueue::SingleFile::Append(base::StringPiece data) { + DCHECK(!is_readonly()); + if (!handle_) { + return Status(error::UNAVAILABLE, base::StrCat({"File not open ", name()})); + } + size_t actual_size = 0; + while (data.size() > 0) { + const int32_t result = handle_->Write(size_, data.data(), data.size()); + if (result < 0) { + return Status( + error::DATA_LOSS, + base::StrCat({"File write error=", + handle_->ErrorToString(handle_->GetLastFileError()), + " ", name()})); + } + size_ += result; + actual_size += result; + data = data.substr(result); // Skip data that has been written. + } + return actual_size; +} + +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_queue.h b/chromium/components/reporting/storage/storage_queue.h new file mode 100644 index 00000000000..a4dd8b7d7ea --- /dev/null +++ b/chromium/components/reporting/storage/storage_queue.h @@ -0,0 +1,348 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_STORAGE_QUEUE_H_ +#define COMPONENTS_REPORTING_STORAGE_STORAGE_QUEUE_H_ + +#include <list> +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "base/callback.h" +#include "base/containers/flat_set.h" +#include "base/files/file.h" +#include "base/files/file_enumerator.h" +#include "base/files/file_path.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "base/optional.h" +#include "base/sequenced_task_runner.h" +#include "base/strings/string_piece.h" +#include "base/threading/thread.h" +#include "base/threading/thread_task_runner_handle.h" +#include "base/timer/timer.h" +#include "components/reporting/encryption/encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/storage/storage_uploader_interface.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// Storage queue represents single queue of data to be collected and stored +// persistently. It allows to add whole data records as necessary, +// flush previously collected records and confirm records up to certain +// sequencing id to be eliminated. +class StorageQueue : public base::RefCountedThreadSafe<StorageQueue> { + public: + // Callback type for UploadInterface provider for this queue. + using StartUploadCb = + base::RepeatingCallback<StatusOr<std::unique_ptr<UploaderInterface>>()>; + + // Creates StorageQueue instance with the specified options, and returns it + // with the |completion_cb| callback. |start_upload_cb| is a factory callback + // that instantiates UploaderInterface every time the queue starts uploading + // records - periodically or immediately after Write (and in the near future - + // upon explicit Flush request). + static void Create( + const QueueOptions& options, + StartUploadCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module, + base::OnceCallback<void(StatusOr<scoped_refptr<StorageQueue>>)> + completion_cb); + + // Wraps and serializes Record (taking ownership of it), encrypts and writes + // the resulting blob into the StorageQueue (the last file of it) with the + // next sequencing id assigned. The write is a non-blocking operation - + // caller can "fire and forget" it (|completion_cb| allows to verify that + // record has been successfully enqueued). If file is going to become too + // large, it is closed and new file is created. + // Helper methods: AssignLastFile, WriteHeaderAndBlock, OpenNewWriteableFile, + // WriteMetadata, DeleteOutdatedMetadata. + void Write(Record record, base::OnceCallback<void(Status)> completion_cb); + + // Confirms acceptance of the records up to |sequencing_id| (inclusively). + // All records with sequencing ids <= this one can be removed from + // the StorageQueue, and can no longer be uploaded. + // If |force| is false (which is used in most cases), |sequencing_id| is + // only accepted if no higher ids were confirmed before; otherwise it is + // accepted unconditionally. + // Helper methods: RemoveConfirmedData. + void Confirm(base::Optional<int64_t> sequencing_id, + bool force, + base::OnceCallback<void(Status)> completion_cb); + + // Initiates upload of collected records. Called periodically by timer, based + // on upload_period of the queue, and can also be called explicitly - for + // a queue with an infinite or very large upload period. Multiple |Flush| + // calls can safely run in parallel. + // Starts by calling |start_upload_cb_| that instantiates |UploaderInterface + // uploader|. Then repeatedly reads EncryptedRecord(s) one by one from the + // StorageQueue starting from |first_sequencing_id_|, handing each one over to + // |uploader|->ProcessRecord (keeping ownership of the buffer) and resuming + // after result callback returns 'true'. Only files that have been closed are + // included in reading; |Upload| makes sure to close the last writeable file + // and create a new one before starting to send records to the |uploader|. + // If some records are not available or corrupt, |uploader|->ProcessGap is + // called. If the monotonic order of sequencing is broken, INTERNAL error + // Status is reported. |Upload| can be stopped after any record by returning + // 'false' to |processed_cb| callback - in that case |Upload| will behave as + // if the end of data has been reached. While one or more |Upload|s are + // active, files can be added to the StorageQueue but cannot be deleted. If + // processing of the record takes significant time, |uploader| implementation + // should be offset to another thread to avoid locking StorageQueue. + // Helper methods: SwitchLastFileIfNotEmpty, CollectFilesForUpload. + void Flush(); + + // Test only: makes specified records fail on reading. + void TestInjectBlockReadErrors(std::initializer_list<int64_t> sequencing_ids); + + // Access queue options. + const QueueOptions& options() const { return options_; } + + StorageQueue(const StorageQueue& other) = delete; + StorageQueue& operator=(const StorageQueue& other) = delete; + + protected: + virtual ~StorageQueue(); + + private: + friend class base::RefCountedThreadSafe<StorageQueue>; + + // Private data structures for Read and Write (need access to the private + // StorageQueue fields). + class WriteContext; + class ReadContext; + class ConfirmContext; + + // Private envelope class for single file in a StorageQueue. + class SingleFile : public base::RefCountedThreadSafe<SingleFile> { + public: + // Factory method creates a SingleFile object for existing + // or new file (of zero size). In case of any error (e.g. insufficient disk + // space) returns status. + static StatusOr<scoped_refptr<SingleFile>> Create( + const base::FilePath& filename, + int64_t size); + + Status Open(bool read_only); // No-op if already opened. + void Close(); // No-op if not opened. + + Status Delete(); + + // Attempts to read |size| bytes from position |pos| and returns + // reference to the data that were actually read (no more than |size|). + // End of file is indicated by empty data. + // |max_buffer_size| specifies the largest allowed buffer, which + // must accommodate the largest possible data block plus header and + // overhead. + StatusOr<base::StringPiece> Read(uint32_t pos, + uint32_t size, + size_t max_buffer_size); + + // Appends data to the file. + StatusOr<uint32_t> Append(base::StringPiece data); + + bool is_opened() const { return handle_.get() != nullptr; } + bool is_readonly() const { + DCHECK(is_opened()); + return is_readonly_.value(); + } + uint64_t size() const { return size_; } + std::string name() const { return filename_.MaybeAsASCII(); } + + protected: + virtual ~SingleFile(); + + private: + friend class base::RefCountedThreadSafe<SingleFile>; + + // Private constructor, called by factory method only. + SingleFile(const base::FilePath& filename, int64_t size); + + // Flag (valid for opened file only): true if file was opened for reading + // only, false otherwise. + base::Optional<bool> is_readonly_; + + const base::FilePath filename_; // relative to the StorageQueue directory + uint64_t size_ = 0; // tracked internally rather than by filesystem + + std::unique_ptr<base::File> handle_; // Set only when opened/created. + + // When reading the file, this is the buffer and data positions. + // If the data is read sequentially, buffered portions are reused + // improving performance. When the sequential order is broken (e.g. + // we start reading the same file in parallel from different position), + // the buffer is reset. + size_t data_start_ = 0; + size_t data_end_ = 0; + uint64_t file_position_ = 0; + size_t buffer_size_ = 0; + std::unique_ptr<char[]> buffer_; + }; + + // Private constructor, to be called by Create factory method only. + StorageQueue(const QueueOptions& options, + StartUploadCb start_upload_cb, + scoped_refptr<EncryptionModule> encryption_module); + + // Initializes the object by enumerating files in the assigned directory + // and determines the sequencing information of the last record. + // Must be called once and only once after construction. + // Returns OK or error status, if anything failed to initialize. + // Called once, during initialization. + // Helper methods: EnumerateDataFiles, ScanLastFile, RestoreMetadata. + Status Init(); + + // Attaches last record digest to the given record (does not exist at a + // generation start). Calculates the given record digest and stores it + // as the last one for the next record. + void UpdateRecordDigest(WrappedRecord* wrapped_record); + + // Helper method for Init(): process single data file. + // Return sequencing_id from <prefix>.<sequencing_id> file name, or Status + // in case there is any error. + StatusOr<int64_t> AddDataFile( + const base::FilePath& full_name, + const base::FileEnumerator::FileInfo& file_info); + + // Helper method for Init(): enumerates all data files in the directory. + // Valid file names are <prefix>.<sequencing_id>, any other names are ignored. + // Adds used data files to the set. + Status EnumerateDataFiles(base::flat_set<base::FilePath>* used_files_set); + + // Helper method for Init(): scans the last file in StorageQueue, if there are + // files at all, and learns the latest sequencing id. Otherwise (if there + // are no files) sets it to 0. + Status ScanLastFile(); + + // Helper method for Write(): increments sequencing id and assigns last + // file to place record in. |size| parameter indicates the size of data that + // comprise the record expected to be appended; if appending the record will + // make the file too large, the current last file will be closed, and a new + // file will be created and assigned to be the last one. + StatusOr<scoped_refptr<SingleFile>> AssignLastFile(size_t size); + + // Helper method for Write() and Read(): creates and opens a new empty + // writeable file, adding it to |files_|. + StatusOr<scoped_refptr<SingleFile>> OpenNewWriteableFile(); + + // Helper method for Write(): stores a file with metadata to match the + // incoming new record. Synchronously composes metadata to record, then + // asynchronously writes it into a file with next sequencing id and then + // notifies the Write operation that it can now complete. After that it + // asynchronously deletes all other files with lower sequencing id + // (multiple Writes can see the same files and attempt to delete them, and + // that is not an error). + Status WriteMetadata(); + + // Helper method for Init(): locates file with metadata that matches the + // last sequencing id and loads metadata from it. + // Adds used metadata file to the set. + Status RestoreMetadata(base::flat_set<base::FilePath>* used_files_set); + + // Delete all files except those listed in the set. + void DeleteUnusedFiles( + const base::flat_set<base::FilePath>& used_files_setused_files_set); + + // Helper method for Write(): deletes meta files up to, but not including + // |sequencing_id_to_keep|. Any errors are ignored. + void DeleteOutdatedMetadata(int64_t sequencing_id_to_keep); + + // Helper method for Write(): composes record header and writes it to the + // file, followed by data. + Status WriteHeaderAndBlock(base::StringPiece data, + scoped_refptr<SingleFile> file); + + // Helper method for Upload: if the last file is not empty (has at least one + // record), close it and create the new one, so that its records are also + // included in the reading. + Status SwitchLastFileIfNotEmpty(); + + // Helper method for Upload: collects and sets aside |files| in the + // StorageQueue that have data for the Upload (all files that have records + // with sequencing ids equal or higher than |sequencing_id|). + std::map<int64_t, scoped_refptr<SingleFile>> CollectFilesForUpload( + int64_t sequencing_id) const; + + // Helper method for Confirm: Moves |first_sequencing_id_| to + // (|sequencing_id|+1) and removes files that only have records with seq + // ids below or equal to |sequencing_id| (below |first_sequencing_id_|). + Status RemoveConfirmedData(int64_t sequencing_id); + + // Helper method to release all file instances held by the queue. + // Files on the disk remain as they were. + void ReleaseAllFileInstances(); + + // Immutable options, stored at the time of creation. + const QueueOptions options_; + + // Current generation id, unique per device and queue. + // Set up once during initialization by reading from the 'gen_id.NNNN' file + // matching the last sequencing id, or generated anew as a random number if no + // such file found (files do not match the id). + int64_t generation_id_ = 0; + + // Digest of the last written record (loaded at queue initialization, absent + // if the new generation has just started, and no records where stored yet). + base::Optional<std::string> last_record_digest_; + + // Queue of the write context instances in the order of creation, sequencing + // ids and record digests. Context is always removed from this queue before + // being destructed. We use std::list rather than std::queue, because + // if the write fails, it needs to be removed from the queue regardless of + // whether it is at the head, tail or middle. + std::list<WriteContext*> write_contexts_queue_; + + // Next sequencing id to store (not assigned yet). + int64_t next_sequencing_id_ = 0; + + // First sequencing id store still has (no records with lower + // sequencing id exist in store). + int64_t first_sequencing_id_ = 0; + + // First unconfirmed sequencing id (no records with lower + // sequencing id will be ever uploaded). Set by the first + // Confirm call. + // If first_unconfirmed_sequencing_id_ < first_sequencing_id_, + // [first_unconfirmed_sequencing_id_, first_sequencing_id_) is a gap + // that cannot be filled in and is uploaded as such. + base::Optional<int64_t> first_unconfirmed_sequencing_id_; + + // Latest metafile. May be null. + scoped_refptr<SingleFile> meta_file_; + + // Ordered map of the files by ascending sequencing id. + std::map<int64_t, scoped_refptr<SingleFile>> files_; + + // Counter of the Read operations. When not 0, none of the files_ can be + // deleted. Incremented by |ReadContext::OnStart|, decremented by + // |ReadContext::OnComplete|. Accessed by |RemoveConfirmedData|. + // All accesses take place on sequenced_task_runner_. + int32_t active_read_operations_ = 0; + + // Upload timer (active only if options_.upload_period() is not 0). + base::RepeatingTimer upload_timer_; + + // Upload provider callback. + const StartUploadCb start_upload_cb_; + + // Encryption module. + scoped_refptr<EncryptionModule> encryption_module_; + + // Test only: records specified to fail on reading. + base::flat_set<int64_t> test_injected_fail_sequencing_ids_; + + // Sequential task runner for all activities in this StorageQueue. + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_; + + SEQUENCE_CHECKER(storage_queue_sequence_checker_); +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_STORAGE_QUEUE_H_ diff --git a/chromium/components/reporting/storage/storage_queue_stress_test.cc b/chromium/components/reporting/storage/storage_queue_stress_test.cc new file mode 100644 index 00000000000..68b1a7880aa --- /dev/null +++ b/chromium/components/reporting/storage/storage_queue_stress_test.cc @@ -0,0 +1,321 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage_queue.h" + +#include <cstdint> +#include <initializer_list> +#include <utility> +#include <vector> + +#include "base/containers/flat_map.h" +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/files/scoped_temp_dir.h" +#include "base/optional.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/synchronization/waitable_event.h" +#include "base/test/task_environment.h" +#include "components/reporting/encryption/test_encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/storage/resources/resource_interface.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" +#include "crypto/sha2.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::_; +using ::testing::Between; +using ::testing::Eq; +using ::testing::Invoke; +using ::testing::NotNull; +using ::testing::Return; +using ::testing::Sequence; +using ::testing::StrEq; +using ::testing::WithArg; + +namespace reporting { +namespace { + +constexpr size_t kTotalQueueStarts = 4; +constexpr size_t kTotalWritesPerStart = 16; +constexpr char kDataPrefix[] = "Rec"; + +// Usage (in tests only): +// +// TestEvent<ResType> e; +// ... Do some async work passing e.cb() as a completion callback of +// base::OnceCallback<void(ResType* res)> type which also may perform some +// other action specified by |done| callback provided by the caller. +// ... = e.result(); // Will wait for e.cb() to be called and return the +// collected result. +// +template <typename ResType> +class TestEvent { + public: + TestEvent() : run_loop_(std::make_unique<base::RunLoop>()) {} + ~TestEvent() { EXPECT_FALSE(run_loop_->running()) << "Not responded"; } + TestEvent(const TestEvent& other) = delete; + TestEvent& operator=(const TestEvent& other) = delete; + ResType result() { + run_loop_->Run(); + return std::forward<ResType>(result_); + } + + // Completion callback to hand over to the processing method. + base::OnceCallback<void(ResType res)> cb() { + return base::BindOnce( + [](base::RunLoop* run_loop, ResType* result, ResType res) { + *result = std::forward<ResType>(res); + run_loop->Quit(); + }, + base::Unretained(run_loop_.get()), base::Unretained(&result_)); + } + + private: + std::unique_ptr<base::RunLoop> run_loop_; + ResType result_; +}; + +class TestUploadClient : public UploaderInterface { + public: + // Mapping of <generation id, sequencing id> to matching record digest. + // Whenever a record is uploaded and includes last record digest, this map + // should have that digest already recorded. Only the first record in a + // generation is uploaded without last record digest. + using LastRecordDigestMap = base::flat_map< + std::pair<int64_t /*generation id */, int64_t /*sequencing id*/>, + base::Optional<std::string /*digest*/>>; + + explicit TestUploadClient(LastRecordDigestMap* last_record_digest_map) + : last_record_digest_map_(last_record_digest_map) {} + + void ProcessRecord(EncryptedRecord encrypted_record, + base::OnceCallback<void(bool)> processed_cb) override { + WrappedRecord wrapped_record; + ASSERT_TRUE(wrapped_record.ParseFromString( + encrypted_record.encrypted_wrapped_record())); + // Verify generation match. + const auto& sequencing_information = + encrypted_record.sequencing_information(); + if (!generation_id_.has_value()) { + generation_id_ = sequencing_information.generation_id(); + } else { + ASSERT_THAT(generation_id_.value(), + Eq(sequencing_information.generation_id())); + } + + // Verify digest and its match. + // Last record digest is not verified yet, since duplicate records are + // accepted in this test. + { + std::string serialized_record; + wrapped_record.record().SerializeToString(&serialized_record); + const auto record_digest = crypto::SHA256HashString(serialized_record); + DCHECK_EQ(record_digest.size(), crypto::kSHA256Length); + ASSERT_THAT(record_digest, Eq(wrapped_record.record_digest())); + // Store record digest for the next record in sequence to verify. + last_record_digest_map_->emplace( + std::make_pair(sequencing_information.sequencing_id(), + sequencing_information.generation_id()), + record_digest); + // If last record digest is present, match it and validate. + if (wrapped_record.has_last_record_digest()) { + auto it = last_record_digest_map_->find( + std::make_pair(sequencing_information.sequencing_id() - 1, + sequencing_information.generation_id())); + if (it != last_record_digest_map_->end() && it->second.has_value()) { + ASSERT_THAT(it->second.value(), + Eq(wrapped_record.last_record_digest())) + << "seq_id=" << sequencing_information.sequencing_id(); + } + } + } + + std::move(processed_cb).Run(true); + } + + void ProcessGap(SequencingInformation sequencing_information, + uint64_t count, + base::OnceCallback<void(bool)> processed_cb) override { + ASSERT_TRUE(false) << "There should be no gaps"; + } + + void Completed(Status status) override { ASSERT_OK(status); } + + private: + base::Optional<int64_t> generation_id_; + LastRecordDigestMap* const last_record_digest_map_; + + Sequence test_upload_sequence_; +}; + +class StorageQueueStressTest : public ::testing::TestWithParam<size_t> { + public: + void SetUp() override { + ASSERT_TRUE(location_.CreateUniqueTempDir()); + options_.set_directory(base::FilePath(location_.GetPath())) + .set_single_file_size(GetParam()); + } + + void TearDown() override { + ResetTestStorageQueue(); + // Make sure all memory is deallocated. + ASSERT_THAT(GetMemoryResource()->GetUsed(), Eq(0u)); + // Make sure all disk is not reserved (files remain, but Storage is not + // responsible for them anymore). + ASSERT_THAT(GetDiskResource()->GetUsed(), Eq(0u)); + } + + void CreateTestStorageQueueOrDie(const QueueOptions& options) { + ASSERT_FALSE(storage_queue_) << "StorageQueue already assigned"; + test_encryption_module_ = + base::MakeRefCounted<test::TestEncryptionModule>(); + TestEvent<StatusOr<scoped_refptr<StorageQueue>>> e; + StorageQueue::Create( + options, + base::BindRepeating(&StorageQueueStressTest::BuildTestUploader, + base::Unretained(this)), + test_encryption_module_, e.cb()); + StatusOr<scoped_refptr<StorageQueue>> storage_queue_result = e.result(); + ASSERT_OK(storage_queue_result) << "Failed to create StorageQueue, error=" + << storage_queue_result.status(); + storage_queue_ = std::move(storage_queue_result.ValueOrDie()); + } + + void ResetTestStorageQueue() { + task_environment_.RunUntilIdle(); + storage_queue_.reset(); + } + + QueueOptions BuildStorageQueueOptionsImmediate() const { + return QueueOptions(options_) + .set_subdirectory(FILE_PATH_LITERAL("D1")) + .set_file_prefix(FILE_PATH_LITERAL("F0001")); + } + + QueueOptions BuildStorageQueueOptionsPeriodic( + base::TimeDelta upload_period = base::TimeDelta::FromSeconds(1)) const { + return BuildStorageQueueOptionsImmediate().set_upload_period(upload_period); + } + + QueueOptions BuildStorageQueueOptionsOnlyManual() const { + return BuildStorageQueueOptionsPeriodic(base::TimeDelta::Max()); + } + + StatusOr<std::unique_ptr<UploaderInterface>> BuildTestUploader() { + return std::make_unique<TestUploadClient>(&last_record_digest_map_); + } + + void WriteStringAsync(base::StringPiece data, + base::OnceCallback<void(Status)> cb) { + EXPECT_TRUE(storage_queue_) << "StorageQueue not created yet"; + Record record; + record.mutable_data()->assign(data.data(), data.size()); + record.set_destination(UPLOAD_EVENTS); + record.set_dm_token("DM TOKEN"); + storage_queue_->Write(std::move(record), std::move(cb)); + } + + base::ScopedTempDir location_; + StorageOptions options_; + scoped_refptr<test::TestEncryptionModule> test_encryption_module_; + scoped_refptr<StorageQueue> storage_queue_; + + // Test-wide global mapping of <generation id, sequencing id> to record + // digest. Serves all TestUploadClients created by test fixture. + TestUploadClient::LastRecordDigestMap last_record_digest_map_; + + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; +}; + +class TestCallbackWaiter { + public: + TestCallbackWaiter() : runner_(base::ThreadTaskRunnerHandle::Get()) {} + TestCallbackWaiter(const TestCallbackWaiter& other) = delete; + TestCallbackWaiter& operator=(const TestCallbackWaiter& other) = delete; + + void Attach() { + const size_t old_counter = counter_.fetch_add(1); + DCHECK_GT(old_counter, 0u) << "Cannot attach when already being released"; + } + + void Signal() { + const size_t old_counter = counter_.fetch_sub(1); + DCHECK_GT(old_counter, 0u) << "Already being released"; + if (old_counter > 1u) { + // There are more owners. + return; + } + // Dropping the last owner. + run_loop_.Quit(); + } + + void Wait() { + Signal(); // Rid of the constructor's ownership. + run_loop_.Run(); + } + + private: + std::atomic<size_t> counter_{1}; // Owned by constructor. + const scoped_refptr<base::SingleThreadTaskRunner> runner_; + base::RunLoop run_loop_; +}; + +TEST_P(StorageQueueStressTest, + WriteIntoNewStorageQueueReopenWriteMoreAndUpload) { + for (size_t iStart = 0; iStart < kTotalQueueStarts; ++iStart) { + TestCallbackWaiter write_waiter; + base::RepeatingCallback<void(Status)> cb = base::BindRepeating( + [](TestCallbackWaiter* waiter, Status status) { + EXPECT_OK(status); + waiter->Signal(); + }, + &write_waiter); + + SCOPED_TRACE(base::StrCat({"Create ", base::NumberToString(iStart)})); + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsOnlyManual()); + + // Write into the queue at random order (simultaneously). + SCOPED_TRACE(base::StrCat({"Write ", base::NumberToString(iStart)})); + const std::string rec_prefix = + base::StrCat({kDataPrefix, base::NumberToString(iStart), "_"}); + for (size_t iRec = 0; iRec < kTotalWritesPerStart; ++iRec) { + write_waiter.Attach(); + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](base::StringPiece rec_prefix, size_t iRec, + StorageQueueStressTest* test, + base::RepeatingCallback<void(Status)> cb) { + test->WriteStringAsync( + base::StrCat({rec_prefix, base::NumberToString(iRec)}), cb); + }, + rec_prefix, iRec, this, cb)); + } + write_waiter.Wait(); + + SCOPED_TRACE(base::StrCat({"Upload ", base::NumberToString(iStart)})); + storage_queue_->Flush(); + + SCOPED_TRACE(base::StrCat({"Reset ", base::NumberToString(iStart)})); + ResetTestStorageQueue(); + EXPECT_THAT(last_record_digest_map_.size(), + Eq((iStart + 1) * kTotalWritesPerStart)); + + SCOPED_TRACE(base::StrCat({"Done ", base::NumberToString(iStart)})); + } +} + +INSTANTIATE_TEST_SUITE_P( + VaryingFileSize, + StorageQueueStressTest, + testing::Values(1 * 1024LL, 2 * 1024LL, 3 * 1024LL, 4 * 1024LL)); + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_queue_unittest.cc b/chromium/components/reporting/storage/storage_queue_unittest.cc new file mode 100644 index 00000000000..be6a4e1a1b9 --- /dev/null +++ b/chromium/components/reporting/storage/storage_queue_unittest.cc @@ -0,0 +1,1093 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage_queue.h" + +#include <cstdint> +#include <initializer_list> +#include <utility> +#include <vector> + +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/files/scoped_temp_dir.h" +#include "base/optional.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/test/task_environment.h" +#include "components/reporting/encryption/test_encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/storage/resources/resource_interface.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" +#include "crypto/sha2.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::_; +using ::testing::Between; +using ::testing::Eq; +using ::testing::Invoke; +using ::testing::NotNull; +using ::testing::Return; +using ::testing::Sequence; +using ::testing::StrEq; +using ::testing::WithArg; + +namespace reporting { +namespace { + +// Metadata file name prefix. +const base::FilePath::CharType METADATA_NAME[] = FILE_PATH_LITERAL("META"); + +// Usage (in tests only): +// +// TestEvent<ResType> e; +// ... Do some async work passing e.cb() as a completion callback of +// base::OnceCallback<void(ResType* res)> type which also may perform some +// other action specified by |done| callback provided by the caller. +// ... = e.result(); // Will wait for e.cb() to be called and return the +// collected result. +// +template <typename ResType> +class TestEvent { + public: + TestEvent() : run_loop_(std::make_unique<base::RunLoop>()) {} + ~TestEvent() { EXPECT_FALSE(run_loop_->running()) << "Not responded"; } + TestEvent(const TestEvent& other) = delete; + TestEvent& operator=(const TestEvent& other) = delete; + ResType result() { + run_loop_->Run(); + return std::forward<ResType>(result_); + } + + // Completion callback to hand over to the processing method. + base::OnceCallback<void(ResType res)> cb() { + return base::BindOnce( + [](base::RunLoop* run_loop, ResType* result, ResType res) { + *result = std::forward<ResType>(res); + run_loop->Quit(); + }, + base::Unretained(run_loop_.get()), base::Unretained(&result_)); + } + + private: + std::unique_ptr<base::RunLoop> run_loop_; + ResType result_; +}; + +class MockUploadClient : public ::testing::NiceMock<UploaderInterface> { + public: + // Mapping of <generation id, sequencing id> to matching record digest. + // Whenever a record is uploaded and includes last record digest, this map + // should have that digest already recorded. Only the first record in a + // generation is uploaded without last record digest. "Optional" is set to + // no-value if there was a gap record instead of a real one. + using LastRecordDigestMap = + std::map<std::pair<int64_t /*generation id */, int64_t /*sequencing id*/>, + base::Optional<std::string /*digest*/>>; + + explicit MockUploadClient(LastRecordDigestMap* last_record_digest_map) + : last_record_digest_map_(last_record_digest_map) {} + + void ProcessRecord(EncryptedRecord encrypted_record, + base::OnceCallback<void(bool)> processed_cb) override { + WrappedRecord wrapped_record; + ASSERT_TRUE(wrapped_record.ParseFromString( + encrypted_record.encrypted_wrapped_record())); + // Verify generation match. + const auto& sequencing_information = + encrypted_record.sequencing_information(); + if (generation_id_.has_value() && + generation_id_.value() != sequencing_information.generation_id()) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.sequencing_id(), + Status( + error::DATA_LOSS, + base::StrCat( + {"Generation id mismatch, expected=", + base::NumberToString(generation_id_.value()), " actual=", + base::NumberToString( + sequencing_information.generation_id())})))); + return; + } + if (!generation_id_.has_value()) { + generation_id_ = sequencing_information.generation_id(); + } + + // Verify digest and its match. + // Last record digest is not verified yet, since duplicate records are + // accepted in this test. + { + std::string serialized_record; + wrapped_record.record().SerializeToString(&serialized_record); + const auto record_digest = crypto::SHA256HashString(serialized_record); + DCHECK_EQ(record_digest.size(), crypto::kSHA256Length); + if (record_digest != wrapped_record.record_digest()) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.sequencing_id(), + Status(error::DATA_LOSS, "Record digest mismatch"))); + return; + } + // Store record digest for the next record in sequence to verify. + last_record_digest_map_->emplace( + std::make_pair(sequencing_information.sequencing_id(), + sequencing_information.generation_id()), + record_digest); + // If last record digest is present, match it and validate. + if (wrapped_record.has_last_record_digest()) { + auto it = last_record_digest_map_->find( + std::make_pair(sequencing_information.sequencing_id() - 1, + sequencing_information.generation_id())); + if (it == last_record_digest_map_->end() || + (it->second.has_value() && + it->second.value() != wrapped_record.last_record_digest())) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.sequencing_id(), + Status(error::DATA_LOSS, "Last record digest mismatch"))); + return; + } + } + } + + EncounterSeqId(sequencing_information.sequencing_id()); + std::move(processed_cb) + .Run(UploadRecord(sequencing_information.sequencing_id(), + wrapped_record.record().data())); + } + + void ProcessGap(SequencingInformation sequencing_information, + uint64_t count, + base::OnceCallback<void(bool)> processed_cb) override { + // Verify generation match. + if (generation_id_.has_value() && + generation_id_.value() != sequencing_information.generation_id()) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.sequencing_id(), + Status( + error::DATA_LOSS, + base::StrCat( + {"Generation id mismatch, expected=", + base::NumberToString(generation_id_.value()), " actual=", + base::NumberToString( + sequencing_information.generation_id())})))); + return; + } + if (!generation_id_.has_value()) { + generation_id_ = sequencing_information.generation_id(); + } + + last_record_digest_map_->emplace( + std::make_pair(sequencing_information.sequencing_id(), + sequencing_information.generation_id()), + base::nullopt); + + for (uint64_t c = 0; c < count; ++c) { + EncounterSeqId(sequencing_information.sequencing_id() + + static_cast<int64_t>(c)); + } + std::move(processed_cb) + .Run(UploadGap(sequencing_information.sequencing_id(), count)); + } + + void Completed(Status status) override { UploadComplete(status); } + + MOCK_METHOD(void, EncounterSeqId, (int64_t), (const)); + MOCK_METHOD(bool, UploadRecord, (int64_t, base::StringPiece), (const)); + MOCK_METHOD(bool, UploadRecordFailure, (int64_t, Status), (const)); + MOCK_METHOD(bool, UploadGap, (int64_t, uint64_t), (const)); + MOCK_METHOD(void, UploadComplete, (Status), (const)); + + // Helper class for setting up mock client expectations of a successful + // completion. + class SetUp { + public: + explicit SetUp(MockUploadClient* client) : client_(client) {} + ~SetUp() { + EXPECT_CALL(*client_, UploadComplete(Eq(Status::StatusOK()))) + .Times(1) + .InSequence(client_->test_upload_sequence_, + client_->test_encounter_sequence_); + } + + SetUp& Required(int64_t sequence_number, base::StringPiece value) { + EXPECT_CALL(*client_, + UploadRecord(Eq(sequence_number), StrEq(std::string(value)))) + .InSequence(client_->test_upload_sequence_) + .WillOnce(Return(true)); + return *this; + } + + SetUp& Possible(int64_t sequence_number, base::StringPiece value) { + EXPECT_CALL(*client_, + UploadRecord(Eq(sequence_number), StrEq(std::string(value)))) + .Times(Between(0, 1)) + .InSequence(client_->test_upload_sequence_) + .WillRepeatedly(Return(true)); + return *this; + } + + SetUp& RequiredGap(int64_t sequence_number, uint64_t count) { + EXPECT_CALL(*client_, UploadGap(Eq(sequence_number), Eq(count))) + .InSequence(client_->test_upload_sequence_) + .WillOnce(Return(true)); + return *this; + } + + SetUp& PossibleGap(int64_t sequence_number, uint64_t count) { + EXPECT_CALL(*client_, UploadGap(Eq(sequence_number), Eq(count))) + .Times(Between(0, 1)) + .InSequence(client_->test_upload_sequence_) + .WillRepeatedly(Return(true)); + return *this; + } + + SetUp& Failure(int64_t sequence_number, Status error) { + EXPECT_CALL(*client_, UploadRecordFailure(Eq(sequence_number), Eq(error))) + .InSequence(client_->test_upload_sequence_) + .WillOnce(Return(true)); + return *this; + } + + // The following two expectations refer to the fact that specific + // sequencing ids have been encountered, regardless of whether they + // belonged to records or gaps. The expectations are set on a separate + // test sequence. + SetUp& RequiredSeqId(int64_t sequence_number) { + EXPECT_CALL(*client_, EncounterSeqId(Eq(sequence_number))) + .Times(1) + .InSequence(client_->test_encounter_sequence_); + return *this; + } + + SetUp& PossibleSeqId(int64_t sequence_number) { + EXPECT_CALL(*client_, EncounterSeqId(Eq(sequence_number))) + .Times(Between(0, 1)) + .InSequence(client_->test_encounter_sequence_); + return *this; + } + + private: + MockUploadClient* const client_; + }; + + private: + base::Optional<int64_t> generation_id_; + LastRecordDigestMap* const last_record_digest_map_; + + Sequence test_encounter_sequence_; + Sequence test_upload_sequence_; +}; + +class StorageQueueTest : public ::testing::TestWithParam<size_t> { + protected: + void SetUp() override { + ASSERT_TRUE(location_.CreateUniqueTempDir()); + options_.set_directory(base::FilePath(location_.GetPath())) + .set_single_file_size(GetParam()); + } + + void TearDown() override { + ResetTestStorageQueue(); + // Make sure all memory is deallocated. + ASSERT_THAT(GetMemoryResource()->GetUsed(), Eq(0u)); + // Make sure all disk is not reserved (files remain, but Storage is not + // responsible for them anymore). + ASSERT_THAT(GetDiskResource()->GetUsed(), Eq(0u)); + } + + void CreateTestStorageQueueOrDie(const QueueOptions& options) { + ASSERT_FALSE(storage_queue_) << "StorageQueue already assigned"; + test_encryption_module_ = + base::MakeRefCounted<test::TestEncryptionModule>(); + TestEvent<StatusOr<scoped_refptr<StorageQueue>>> e; + StorageQueue::Create( + options, + base::BindRepeating(&StorageQueueTest::BuildMockUploader, + base::Unretained(this)), + test_encryption_module_, e.cb()); + StatusOr<scoped_refptr<StorageQueue>> storage_queue_result = e.result(); + ASSERT_OK(storage_queue_result) << "Failed to create StorageQueue, error=" + << storage_queue_result.status(); + storage_queue_ = std::move(storage_queue_result.ValueOrDie()); + } + + void ResetTestStorageQueue() { + task_environment_.RunUntilIdle(); + storage_queue_.reset(); + } + + void InjectFailures(std::initializer_list<int64_t> sequencing_ids) { + storage_queue_->TestInjectBlockReadErrors(sequencing_ids); + } + + QueueOptions BuildStorageQueueOptionsImmediate() const { + return QueueOptions(options_) + .set_subdirectory(FILE_PATH_LITERAL("D1")) + .set_file_prefix(FILE_PATH_LITERAL("F0001")); + } + + QueueOptions BuildStorageQueueOptionsPeriodic( + base::TimeDelta upload_period = base::TimeDelta::FromSeconds(1)) const { + return BuildStorageQueueOptionsImmediate().set_upload_period(upload_period); + } + + QueueOptions BuildStorageQueueOptionsOnlyManual() const { + return BuildStorageQueueOptionsPeriodic(base::TimeDelta::Max()); + } + + StatusOr<std::unique_ptr<UploaderInterface>> BuildMockUploader() { + auto uploader = + std::make_unique<MockUploadClient>(&last_record_digest_map_); + set_mock_uploader_expectations_.Call(uploader.get()); + return uploader; + } + + Status WriteString(base::StringPiece data) { + EXPECT_TRUE(storage_queue_) << "StorageQueue not created yet"; + TestEvent<Status> w; + Record record; + record.set_data(std::string(data)); + record.set_destination(UPLOAD_EVENTS); + record.set_dm_token("DM TOKEN"); + storage_queue_->Write(std::move(record), w.cb()); + return w.result(); + } + + void WriteStringOrDie(base::StringPiece data) { + const Status write_result = WriteString(data); + ASSERT_OK(write_result) << write_result; + } + + void ConfirmOrDie(base::Optional<std::int64_t> sequencing_id, + bool force = false) { + TestEvent<Status> c; + storage_queue_->Confirm(sequencing_id, force, c.cb()); + const Status c_result = c.result(); + ASSERT_OK(c_result) << c_result; + } + + base::ScopedTempDir location_; + StorageOptions options_; + scoped_refptr<test::TestEncryptionModule> test_encryption_module_; + scoped_refptr<StorageQueue> storage_queue_; + + // Test-wide global mapping of <generation id, sequencing id> to record + // digest. Serves all MockUploadClients created by test fixture. + MockUploadClient::LastRecordDigestMap last_record_digest_map_; + + ::testing::MockFunction<void(MockUploadClient*)> + set_mock_uploader_expectations_; + + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; +}; + +constexpr std::array<const char*, 3> kData = {"Rec1111", "Rec222", "Rec33"}; +constexpr std::array<const char*, 3> kMoreData = {"More1111", "More222", + "More33"}; + +TEST_P(StorageQueueTest, WriteIntoNewStorageQueueAndReopen) { + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())).Times(0); + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + ResetTestStorageQueue(); + + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); +} + +TEST_P(StorageQueueTest, WriteIntoNewStorageQueueReopenAndWriteMore) { + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())).Times(0); + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + ResetTestStorageQueue(); + + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); +} + +TEST_P(StorageQueueTest, WriteIntoNewStorageQueueAndUpload) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + + // Trigger upload. + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, WriteIntoNewStorageQueueAndUploadWithFailures) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Inject simulated failures. + InjectFailures({1}); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .RequiredGap(1, 1) + .Possible(2, kData[2]); // Depending on records binpacking + })); + + // Trigger upload. + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, WriteIntoNewStorageQueueReopenWriteMoreAndUpload) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + ResetTestStorageQueue(); + + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + + // Trigger upload. + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, + WriteIntoNewStorageQueueReopenWithMissingMetadataWriteMoreAndUpload) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Save copy of options. + const QueueOptions options = storage_queue_->options(); + + ResetTestStorageQueue(); + + // Delete all metadata files. + base::FileEnumerator dir_enum( + options.directory(), + /*recursive=*/false, base::FileEnumerator::FILES, + base::StrCat({METADATA_NAME, FILE_PATH_LITERAL(".*")})); + base::FilePath full_name; + while (full_name = dir_enum.Next(), !full_name.empty()) { + base::DeleteFile(full_name); + } + + // Reopen, starting a new generation. + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); + + // Set uploader expectations. Previous data is all lost. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kMoreData[0]) + .Required(1, kMoreData[1]) + .Required(2, kMoreData[2]); + })); + + // Trigger upload. + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, + WriteIntoNewStorageQueueReopenWithMissingDataWriteMoreAndUpload) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Save copy of options. + const QueueOptions options = storage_queue_->options(); + + ResetTestStorageQueue(); + + // Reopen with the same generation and sequencing information. + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + + // Delete the first data file. + base::FilePath full_name = options.directory().Append( + base::StrCat({options.file_prefix(), FILE_PATH_LITERAL(".0")})); + base::DeleteFile(full_name); + + // Write more data. + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); + + // Set uploader expectations. Previous data is all lost. + // The expected results depend on the test configuration. + switch (options.single_file_size()) { + case 1: // single record in file - deletion killed the first record + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .PossibleGap(0, 1) + .Required(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + break; + case 256: // two records in file - deletion killed the first two records. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .PossibleGap(0, 2) + .Failure( + 2, Status(error::DATA_LOSS, "Last record digest mismatch")) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + break; + default: // UNlimited file size - deletion above killed all the data. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client).PossibleGap(0, 1); + })); + } + + // Trigger upload. + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, WriteIntoNewStorageQueueAndFlush) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsOnlyManual()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + + // Flush manually. + storage_queue_->Flush(); +} + +TEST_P(StorageQueueTest, WriteIntoNewStorageQueueReopenWriteMoreAndFlush) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsOnlyManual()); + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + ResetTestStorageQueue(); + + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsOnlyManual()); + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + + // Flush manually. + storage_queue_->Flush(); +} + +TEST_P(StorageQueueTest, ValidateVariousRecordSizes) { + std::vector<std::string> data; + for (size_t i = 16; i < 16 + 16; ++i) { + data.emplace_back(i, 'R'); + } + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsOnlyManual()); + for (const auto& record : data) { + WriteStringOrDie(record); + } + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([&data](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp client_setup(mock_upload_client); + for (size_t i = 0; i < data.size(); ++i) { + client_setup.Required(i, data[i]); + } + })); + + // Flush manually. + storage_queue_->Flush(); +} + +TEST_P(StorageQueueTest, WriteAndRepeatedlyUploadWithConfirmations) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #0 and forward time again, removing record #0 + ConfirmOrDie(/*sequencing_id=*/0); + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #1 and forward time again, removing record #1 + ConfirmOrDie(/*sequencing_id=*/1); + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client).Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Add more data and verify that #2 and new data are returned. + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #2 and forward time again, removing record #2 + ConfirmOrDie(/*sequencing_id=*/2); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, WriteAndRepeatedlyUploadWithConfirmationsAndReopen) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #0 and forward time again, removing record #0 + ConfirmOrDie(/*sequencing_id=*/0); + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #1 and forward time again, removing record #1 + ConfirmOrDie(/*sequencing_id=*/1); + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client).Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + ResetTestStorageQueue(); + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + + // Add more data and verify that #2 and new data are returned. + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #2 and forward time again, removing record #2 + ConfirmOrDie(/*sequencing_id=*/2); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, + WriteAndRepeatedlyUploadWithConfirmationsAndReopenWithFailures) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #0 and forward time again, removing record #0 + ConfirmOrDie(/*sequencing_id=*/0); + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #1 and forward time again, removing record #1 + ConfirmOrDie(/*sequencing_id=*/1); + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client).Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + ResetTestStorageQueue(); + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + + // Add more data and verify that #2 and new data are returned. + WriteStringOrDie(kMoreData[0]); + WriteStringOrDie(kMoreData[1]); + WriteStringOrDie(kMoreData[2]); + + // Inject simulated failures. + InjectFailures({4, 5}); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + // Gap may be 2 records at once or 2 gaps 1 record each. + .PossibleGap(4, 2) + .PossibleGap(4, 1) + .PossibleGap(5, 1); + })); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #2 and forward time again, removing record #2 + ConfirmOrDie(/*sequencing_id=*/2); + + // Reset simulated failures. + InjectFailures({}); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageQueueTest, WriteAndRepeatedlyImmediateUpload) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsImmediate()); + + // Upload is initiated asynchronously, so it may happen after the next + // record is also written. Because of that we set expectations for the + // data after the current one as |Possible|. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + })); + WriteStringOrDie(kData[0]); + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Possible(2, kData[2]); + })); + WriteStringOrDie(kData[1]); + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + WriteStringOrDie(kData[2]); +} + +TEST_P(StorageQueueTest, WriteAndRepeatedlyImmediateUploadWithConfirmations) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsImmediate()); + + // Upload is initiated asynchronously, so it may happen after the next + // record is also written. Because of the Confirmation below, we set + // expectations for the data that may be eliminated by Confirmation as + // |Possible|. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + })); + WriteStringOrDie(kData[0]); + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + })); + WriteStringOrDie(kData[1]); + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Required(2, kData[2]); // Not confirmed - hence |Required| + })); + WriteStringOrDie(kData[2]); + + // Confirm #1, removing data #0 and #1 + ConfirmOrDie(/*sequencing_id=*/1); + + // Add more data and verify that #2 and new data are returned. + // Upload is initiated asynchronously, so it may happen after the next + // record is also written. Because of that we set expectations for the + // data after the current one as |Possible|. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Possible(4, kMoreData[1]) + .Possible(5, kMoreData[2]); + })); + WriteStringOrDie(kMoreData[0]); + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Possible(5, kMoreData[2]); + })); + WriteStringOrDie(kMoreData[1]); + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + })); + WriteStringOrDie(kMoreData[2]); +} + +TEST_P(StorageQueueTest, WriteEncryptFailure) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + DCHECK(test_encryption_module_); + EXPECT_CALL(*test_encryption_module_, EncryptRecord(_, _)) + .WillOnce(WithArg<1>( + Invoke([](base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb) { + std::move(cb).Run(Status(error::UNKNOWN, "Failing for tests")); + }))); + const Status result = WriteString("TEST_MESSAGE"); + EXPECT_FALSE(result.ok()); + EXPECT_EQ(result.error_code(), error::UNKNOWN); +} + +TEST_P(StorageQueueTest, ForceConfirm) { + CreateTestStorageQueueOrDie(BuildStorageQueueOptionsPeriodic()); + + WriteStringOrDie(kData[0]); + WriteStringOrDie(kData[1]); + WriteStringOrDie(kData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #1 and forward time again, possibly removing records #0 and #1 + ConfirmOrDie(/*sequencing_id=*/1); + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client).Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Now force confirm the very beginning and forward time again. + ConfirmOrDie(/*sequencing_id=*/base::nullopt, /*force=*/true); + // Set uploader expectations: #0 and #1 could be returned as Gaps + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .RequiredSeqId(0) + .RequiredSeqId(1) + .RequiredSeqId(2) + // 0-2 must have been encountered, but actual contents + // can be different: + .Possible(0, kData[0]) + .PossibleGap(0, 1) + .PossibleGap(0, 2) + .Possible(1, kData[1]) + .Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Force confirm #0 and forward time again. + ConfirmOrDie(/*sequencing_id=*/0, /*force=*/true); + // Set uploader expectations: #0 and #1 could be returned as Gaps + EXPECT_CALL(set_mock_uploader_expectations_, Call(NotNull())) + .WillOnce(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(mock_upload_client) + .RequiredSeqId(1) + .RequiredSeqId(2) + // 0-2 must have been encountered, but actual contents + // can be different: + .PossibleGap(1, 1) + .Possible(1, kData[1]) + .Required(2, kData[2]); + })); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +INSTANTIATE_TEST_SUITE_P(VaryingFileSize, + StorageQueueTest, + testing::Values(128 * 1024LL * 1024LL, + 256 /* two records in file */, + 1 /* single record in file */)); + +// TODO(b/157943006): Additional tests: +// 1) Options object with a bad path. +// 2) Have bad prefix files in the directory. +// 3) Attempt to create file with duplicated file extension. +// 4) Disk and memory limit exceeded. +// 5) Other negative tests. + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_unittest.cc b/chromium/components/reporting/storage/storage_unittest.cc new file mode 100644 index 00000000000..14c576bbf6b --- /dev/null +++ b/chromium/components/reporting/storage/storage_unittest.cc @@ -0,0 +1,1293 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage.h" + +#include <cstdint> +#include <tuple> +#include <utility> + +#include "base/files/scoped_temp_dir.h" +#include "base/optional.h" +#include "base/sequenced_task_runner.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/test/scoped_feature_list.h" +#include "base/test/task_environment.h" +#include "components/reporting/encryption/decryption.h" +#include "components/reporting/encryption/encryption.h" +#include "components/reporting/encryption/test_encryption_module.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/storage/resources/resource_interface.h" +#include "components/reporting/storage/storage_configuration.h" +#include "components/reporting/storage/storage_uploader_interface.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/status_macros.h" +#include "components/reporting/util/statusor.h" +#include "crypto/sha2.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/boringssl/src/include/openssl/curve25519.h" + +using ::testing::_; +using ::testing::Between; +using ::testing::Eq; +using ::testing::Invoke; +using ::testing::Ne; +using ::testing::NotNull; +using ::testing::Property; +using ::testing::Return; +using ::testing::Sequence; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +namespace reporting { +namespace { + +// Usage (in tests only): +// +// TestEvent<ResType> e; +// ... Do some async work passing e.cb() as a completion callback of +// base::OnceCallback<void(ResType* res)> type which also may perform some +// other action specified by |done| callback provided by the caller. +// ... = e.result(); // Will wait for e.cb() to be called and return the +// collected result. +// +template <typename ResType> +class TestEvent { + public: + TestEvent() : run_loop_(std::make_unique<base::RunLoop>()) {} + ~TestEvent() { EXPECT_FALSE(run_loop_->running()) << "Not responded"; } + TestEvent(const TestEvent& other) = delete; + TestEvent& operator=(const TestEvent& other) = delete; + ResType result() { + run_loop_->Run(); + return std::forward<ResType>(result_); + } + + // Completion callback to hand over to the processing method. + base::OnceCallback<void(ResType res)> cb() { + return base::BindOnce( + [](base::RunLoop* run_loop, ResType* result, ResType res) { + *result = std::forward<ResType>(res); + run_loop->Quit(); + }, + base::Unretained(run_loop_.get()), base::Unretained(&result_)); + } + + private: + std::unique_ptr<base::RunLoop> run_loop_; + ResType result_; +}; + +// Context of single decryption. Self-destructs upon completion or failure. +class SingleDecryptionContext { + public: + SingleDecryptionContext( + const EncryptedRecord& encrypted_record, + scoped_refptr<Decryptor> decryptor, + base::OnceCallback<void(StatusOr<base::StringPiece>)> response) + : encrypted_record_(encrypted_record), + decryptor_(decryptor), + response_(std::move(response)) {} + + SingleDecryptionContext(const SingleDecryptionContext& other) = delete; + SingleDecryptionContext& operator=(const SingleDecryptionContext& other) = + delete; + + ~SingleDecryptionContext() { + DCHECK(!response_) << "Self-destruct without prior response"; + } + + void Start() { + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::RetrieveMatchingPrivateKey, + base::Unretained(this))); + } + + private: + void Respond(StatusOr<base::StringPiece> result) { + std::move(response_).Run(result); + delete this; + } + + void RetrieveMatchingPrivateKey() { + // Retrieve private key that matches public key hash. + decryptor_->RetrieveMatchingPrivateKey( + encrypted_record_.encryption_info().public_key_id(), + base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<std::string> private_key_result) { + if (!private_key_result.ok()) { + self->Respond(private_key_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::DecryptSharedSecret, + base::Unretained(self), + private_key_result.ValueOrDie())); + }, + base::Unretained(this))); + } + + void DecryptSharedSecret(base::StringPiece private_key) { + // Decrypt shared secret from private key and peer public key. + auto shared_secret_result = decryptor_->DecryptSecret( + private_key, encrypted_record_.encryption_info().encryption_key()); + if (!shared_secret_result.ok()) { + Respond(shared_secret_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, base::BindOnce(&SingleDecryptionContext::OpenRecord, + base::Unretained(this), + shared_secret_result.ValueOrDie())); + } + + void OpenRecord(base::StringPiece shared_secret) { + decryptor_->OpenRecord( + shared_secret, + base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<Decryptor::Handle*> handle_result) { + if (!handle_result.ok()) { + self->Respond(handle_result.status()); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::AddToRecord, + base::Unretained(self), + base::Unretained(handle_result.ValueOrDie()))); + }, + base::Unretained(this))); + } + + void AddToRecord(Decryptor::Handle* handle) { + handle->AddToRecord( + encrypted_record_.encrypted_wrapped_record(), + base::BindOnce( + [](SingleDecryptionContext* self, Decryptor::Handle* handle, + Status status) { + if (!status.ok()) { + self->Respond(status); + return; + } + base::ThreadPool::PostTask( + FROM_HERE, + base::BindOnce(&SingleDecryptionContext::CloseRecord, + base::Unretained(self), + base::Unretained(handle))); + }, + base::Unretained(this), base::Unretained(handle))); + } + + void CloseRecord(Decryptor::Handle* handle) { + handle->CloseRecord(base::BindOnce( + [](SingleDecryptionContext* self, + StatusOr<base::StringPiece> decryption_result) { + self->Respond(decryption_result); + }, + base::Unretained(this))); + } + + private: + const EncryptedRecord encrypted_record_; + const scoped_refptr<Decryptor> decryptor_; + base::OnceCallback<void(StatusOr<base::StringPiece>)> response_; +}; + +class MockUploadClient : public ::testing::NiceMock<UploaderInterface> { + public: + // Mapping of <generation id, sequencing id> to matching record digest. + // Whenever a record is uploaded and includes last record digest, this map + // should have that digest already recorded. Only the first record in a + // generation is uploaded without last record digest. + using LastRecordDigestMap = std::map<std::tuple<Priority, + int64_t /*generation id*/, + int64_t /*sequencing id*/>, + base::Optional<std::string /*digest*/>>; + + explicit MockUploadClient( + LastRecordDigestMap* last_record_digest_map, + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner, + scoped_refptr<Decryptor> decryptor) + : last_record_digest_map_(last_record_digest_map), + sequenced_task_runner_(sequenced_task_runner), + decryptor_(decryptor) {} + + void ProcessRecord(EncryptedRecord encrypted_record, + base::OnceCallback<void(bool)> processed_cb) override { + const auto& sequencing_information = + encrypted_record.sequencing_information(); + if (!encrypted_record.has_encryption_info()) { + // Wrapped record is not encrypted. + WrappedRecord wrapped_record; + ASSERT_TRUE(wrapped_record.ParseFromString( + encrypted_record.encrypted_wrapped_record())); + ScheduleVerifyRecord(sequencing_information, std::move(wrapped_record), + std::move(processed_cb)); + return; + } + // Decrypt encrypted_record. + (new SingleDecryptionContext( + encrypted_record, decryptor_, + base::BindOnce( + [](SequencingInformation sequencing_information, + base::OnceCallback<void(bool)> processed_cb, + MockUploadClient* client, StatusOr<base::StringPiece> result) { + ASSERT_OK(result.status()); + WrappedRecord wrapped_record; + ASSERT_TRUE(wrapped_record.ParseFromArray( + result.ValueOrDie().data(), result.ValueOrDie().size())); + // Verify wrapped record once decrypted. + client->ScheduleVerifyRecord(sequencing_information, + std::move(wrapped_record), + std::move(processed_cb)); + }, + sequencing_information, std::move(processed_cb), + base::Unretained(this)))) + ->Start(); + } + + void ProcessGap(SequencingInformation sequencing_information, + uint64_t count, + base::OnceCallback<void(bool)> processed_cb) override { + // Verify generation match. + if (generation_id_.has_value() && + generation_id_.value() != sequencing_information.generation_id()) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.priority(), + sequencing_information.sequencing_id(), + Status( + error::DATA_LOSS, + base::StrCat( + {"Generation id mismatch, expected=", + base::NumberToString(generation_id_.value()), " actual=", + base::NumberToString( + sequencing_information.generation_id())})))); + return; + } + if (!generation_id_.has_value()) { + generation_id_ = sequencing_information.generation_id(); + } + + last_record_digest_map_->emplace( + std::make_tuple(sequencing_information.priority(), + sequencing_information.sequencing_id(), + sequencing_information.generation_id()), + base::nullopt); + + for (uint64_t c = 0; c < count; ++c) { + EncounterSeqId( + sequencing_information.priority(), + sequencing_information.sequencing_id() + static_cast<int64_t>(c)); + } + std::move(processed_cb) + .Run(UploadGap(sequencing_information.priority(), + sequencing_information.sequencing_id(), count)); + } + + void Completed(Status status) override { UploadComplete(status); } + + MOCK_METHOD(void, EncounterSeqId, (Priority, int64_t), (const)); + MOCK_METHOD(bool, + UploadRecord, + (Priority, int64_t, base::StringPiece), + (const)); + MOCK_METHOD(bool, UploadRecordFailure, (Priority, int64_t, Status), (const)); + MOCK_METHOD(bool, UploadGap, (Priority, int64_t, uint64_t), (const)); + MOCK_METHOD(void, UploadComplete, (Status), (const)); + + // Helper class for setting up mock client expectations of a successful + // completion. + class SetUp { + public: + SetUp(Priority priority, MockUploadClient* client) + : priority_(priority), client_(client) {} + + ~SetUp() { + EXPECT_CALL(*client_, UploadRecordFailure(_, _, _)) + .Times(0) + .InSequence(client_->test_upload_sequence_); + EXPECT_CALL(*client_, UploadComplete(Eq(Status::StatusOK()))) + .Times(1) + .InSequence(client_->test_upload_sequence_, + client_->test_encounter_sequence_); + } + + SetUp& Required(int64_t sequencing_id, base::StringPiece value) { + EXPECT_CALL(*client_, UploadRecord(Eq(priority_), Eq(sequencing_id), + StrEq(std::string(value)))) + .InSequence(client_->test_upload_sequence_) + .WillOnce(Return(true)); + return *this; + } + + SetUp& Possible(int64_t sequencing_id, base::StringPiece value) { + EXPECT_CALL(*client_, UploadRecord(Eq(priority_), Eq(sequencing_id), + StrEq(std::string(value)))) + .Times(Between(0, 1)) + .InSequence(client_->test_upload_sequence_) + .WillRepeatedly(Return(true)); + return *this; + } + + SetUp& PossibleGap(int64_t sequence_number, uint64_t count) { + EXPECT_CALL(*client_, + UploadGap(Eq(priority_), Eq(sequence_number), Eq(count))) + .Times(Between(0, 1)) + .InSequence(client_->test_upload_sequence_) + .WillRepeatedly(Return(true)); + return *this; + } + + // The following two expectations refer to the fact that specific + // sequencing ids have been encountered, regardless of whether they + // belonged to records or gaps. The expectations are set on a separate + // test sequence. + SetUp& RequiredSeqId(int64_t sequence_number) { + EXPECT_CALL(*client_, EncounterSeqId(Eq(priority_), Eq(sequence_number))) + .Times(1) + .InSequence(client_->test_encounter_sequence_); + return *this; + } + + SetUp& PossibleSeqId(int64_t sequence_number) { + EXPECT_CALL(*client_, EncounterSeqId(Eq(priority_), Eq(sequence_number))) + .Times(Between(0, 1)) + .InSequence(client_->test_encounter_sequence_); + return *this; + } + + private: + Priority priority_; + MockUploadClient* const client_; + }; + + // Helper class for setting up mock client expectations on empty queue. + class SetEmpty { + public: + explicit SetEmpty(MockUploadClient* client) : client_(client) {} + + ~SetEmpty() { + EXPECT_CALL(*client_, UploadRecord(_, _, _)).Times(0); + EXPECT_CALL(*client_, UploadRecordFailure(_, _, _)).Times(0); + EXPECT_CALL(*client_, UploadComplete(Property(&Status::error_code, + Eq(error::OUT_OF_RANGE)))) + .Times(1); + } + + private: + MockUploadClient* const client_; + }; + + // Helper class for setting up mock client expectations for key delivery. + class SetKeyDelivery { + public: + explicit SetKeyDelivery(MockUploadClient* client) : client_(client) {} + + ~SetKeyDelivery() { + EXPECT_CALL(*client_, UploadRecord(_, _, _)).Times(0); + EXPECT_CALL(*client_, UploadRecordFailure(_, _, _)).Times(0); + EXPECT_CALL(*client_, UploadComplete(Eq(Status::StatusOK()))).Times(1); + } + + private: + MockUploadClient* const client_; + }; + + private: + void ScheduleVerifyRecord(SequencingInformation sequencing_information, + WrappedRecord wrapped_record, + base::OnceCallback<void(bool)> processed_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&MockUploadClient::VerifyRecord, base::Unretained(this), + sequencing_information, std::move(wrapped_record), + std::move(processed_cb))); + } + + void VerifyRecord(SequencingInformation sequencing_information, + WrappedRecord wrapped_record, + base::OnceCallback<void(bool)> processed_cb) { + // Verify generation match. + if (generation_id_.has_value() && + generation_id_.value() != sequencing_information.generation_id()) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.priority(), + sequencing_information.sequencing_id(), + Status( + error::DATA_LOSS, + base::StrCat( + {"Generation id mismatch, expected=", + base::NumberToString(generation_id_.value()), " actual=", + base::NumberToString( + sequencing_information.generation_id())})))); + return; + } + if (!generation_id_.has_value()) { + generation_id_ = sequencing_information.generation_id(); + } + + // Verify digest and its match. + // Last record digest is not verified yet, since duplicate records are + // accepted in this test. + { + std::string serialized_record; + wrapped_record.record().SerializeToString(&serialized_record); + const auto record_digest = crypto::SHA256HashString(serialized_record); + DCHECK_EQ(record_digest.size(), crypto::kSHA256Length); + if (record_digest != wrapped_record.record_digest()) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.priority(), + sequencing_information.sequencing_id(), + Status(error::DATA_LOSS, "Record digest mismatch"))); + return; + } + if (wrapped_record.has_last_record_digest()) { + auto it = last_record_digest_map_->find( + std::make_tuple(sequencing_information.priority(), + sequencing_information.sequencing_id() - 1, + sequencing_information.generation_id())); + if (it == last_record_digest_map_->end() || + it->second != wrapped_record.last_record_digest()) { + std::move(processed_cb) + .Run(UploadRecordFailure( + sequencing_information.priority(), + sequencing_information.sequencing_id(), + Status(error::DATA_LOSS, "Last record digest mismatch"))); + return; + } + } + last_record_digest_map_->emplace( + std::make_tuple(sequencing_information.priority(), + sequencing_information.sequencing_id(), + sequencing_information.generation_id()), + record_digest); + } + + EncounterSeqId(sequencing_information.priority(), + sequencing_information.sequencing_id()); + std::move(processed_cb) + .Run(UploadRecord(sequencing_information.priority(), + sequencing_information.sequencing_id(), + wrapped_record.record().data())); + } + + base::Optional<int64_t> generation_id_; + LastRecordDigestMap* const last_record_digest_map_; + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_; + + const scoped_refptr<Decryptor> decryptor_; + + Sequence test_encounter_sequence_; + Sequence test_upload_sequence_; +}; + +class StorageTest + : public ::testing::TestWithParam<::testing::tuple<bool, size_t>> { + protected: + void SetUp() override { + ASSERT_TRUE(location_.CreateUniqueTempDir()); + // Encryption is disabled by default. + ASSERT_FALSE(EncryptionModule::is_enabled()); + if (is_encryption_enabled()) { + // Enable encryption. + scoped_feature_list_.InitFromCommandLine( + {EncryptionModule::kEncryptedReporting}, {}); + // Generate signing key pair. + ED25519_keypair(signature_verification_public_key_, signing_private_key_); + // Create decryption module. + auto decryptor_result = Decryptor::Create(); + ASSERT_OK(decryptor_result.status()) << decryptor_result.status(); + decryptor_ = std::move(decryptor_result.ValueOrDie()); + // First creation of Storage would need key delivered. + expect_to_need_key_ = true; + } + } + + void TearDown() override { + ResetTestStorage(); + // Make sure all memory is deallocated. + ASSERT_THAT(GetMemoryResource()->GetUsed(), Eq(0u)); + // Make sure all disk is not reserved (files remain, but Storage is not + // responsible for them anymore). + ASSERT_THAT(GetDiskResource()->GetUsed(), Eq(0u)); + } + + StatusOr<scoped_refptr<Storage>> CreateTestStorage( + const StorageOptions& options, + scoped_refptr<EncryptionModule> encryption_module) { + if (expect_to_need_key_) { + // Set uploader expectations for any queue; expect no records and need + // key. Make sure no uploads happen, and key is requested. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(_, /*need_encryption_key=*/Eq(true), NotNull())) + .WillOnce(WithArg<2>(Invoke([](MockUploadClient* mock_upload_client) { + MockUploadClient::SetKeyDelivery client(mock_upload_client); + }))) + .RetiresOnSaturation(); + } + // Initialize Storage with no key. + TestEvent<StatusOr<scoped_refptr<Storage>>> e; + Storage::Create(options, + base::BindRepeating(&StorageTest::BuildMockUploader, + base::Unretained(this)), + encryption_module, e.cb()); + ASSIGN_OR_RETURN(auto storage, e.result()); + if (expect_to_need_key_) { + // Provision the storage with a key. + // Key delivery must have been requested above. + GenerateAndDeliverKey(storage.get()); + } + return storage; + } + + void CreateTestStorageOrDie( + const StorageOptions& options, + scoped_refptr<EncryptionModule> encryption_module = + base::MakeRefCounted<EncryptionModule>( + /*renew_encryption_key_period=*/base::TimeDelta::FromMinutes( + 30))) { + ASSERT_FALSE(storage_) << "StorageTest already assigned"; + StatusOr<scoped_refptr<Storage>> storage_result = + CreateTestStorage(options, encryption_module); + ASSERT_OK(storage_result) + << "Failed to create StorageTest, error=" << storage_result.status(); + storage_ = std::move(storage_result.ValueOrDie()); + } + + void ResetTestStorage() { + task_environment_.RunUntilIdle(); + storage_.reset(); + expect_to_need_key_ = false; + } + + StorageOptions BuildTestStorageOptions() const { + auto options = StorageOptions() + .set_directory(base::FilePath(location_.GetPath())) + .set_single_file_size(is_encryption_enabled()); + if (is_encryption_enabled()) { + // Encryption enabled. + options.set_signature_verification_public_key(std::string( + reinterpret_cast<const char*>(signature_verification_public_key_), + ED25519_PUBLIC_KEY_LEN)); + } + return options; + } + + StatusOr<std::unique_ptr<UploaderInterface>> BuildMockUploader( + Priority priority, + bool need_encryption_key) { + auto uploader = std::make_unique<MockUploadClient>( + &last_record_digest_map_, sequenced_task_runner_, decryptor_); + set_mock_uploader_expectations_.Call(priority, need_encryption_key, + uploader.get()); + return uploader; + } + + Status WriteString(Priority priority, base::StringPiece data) { + EXPECT_TRUE(storage_) << "Storage not created yet"; + TestEvent<Status> w; + Record record; + record.set_data(std::string(data)); + record.set_destination(UPLOAD_EVENTS); + record.set_dm_token("DM TOKEN"); + storage_->Write(priority, std::move(record), w.cb()); + return w.result(); + } + + void WriteStringOrDie(Priority priority, base::StringPiece data) { + const Status write_result = WriteString(priority, data); + ASSERT_OK(write_result) << write_result; + } + + void ConfirmOrDie(Priority priority, + base::Optional<std::int64_t> sequencing_id, + bool force = false) { + TestEvent<Status> c; + storage_->Confirm(priority, sequencing_id, force, c.cb()); + const Status c_result = c.result(); + ASSERT_OK(c_result) << c_result; + } + + void GenerateAndDeliverKey(Storage* storage) { + ASSERT_TRUE(decryptor_) << "Decryptor not created"; + // Generate new pair of private key and public value. + uint8_t private_key[X25519_PRIVATE_KEY_LEN]; + Encryptor::PublicKeyId public_key_id; + uint8_t public_value[X25519_PUBLIC_VALUE_LEN]; + X25519_keypair(public_value, private_key); + TestEvent<StatusOr<Encryptor::PublicKeyId>> prepare_key_pair; + decryptor_->RecordKeyPair( + std::string(reinterpret_cast<const char*>(private_key), + X25519_PRIVATE_KEY_LEN), + std::string(reinterpret_cast<const char*>(public_value), + X25519_PUBLIC_VALUE_LEN), + prepare_key_pair.cb()); + auto prepare_key_result = prepare_key_pair.result(); + ASSERT_OK(prepare_key_result.status()); + public_key_id = prepare_key_result.ValueOrDie(); + // Deliver public key to storage. + SignedEncryptionInfo signed_encryption_key; + signed_encryption_key.set_public_asymmetric_key(std::string( + reinterpret_cast<const char*>(public_value), X25519_PUBLIC_VALUE_LEN)); + signed_encryption_key.set_public_key_id(public_key_id); + // Sign public key. + uint8_t + value_to_sign[sizeof(Encryptor::PublicKeyId) + X25519_PUBLIC_VALUE_LEN]; + memcpy(value_to_sign, &public_key_id, sizeof(Encryptor::PublicKeyId)); + memcpy(value_to_sign + sizeof(Encryptor::PublicKeyId), public_value, + X25519_PUBLIC_VALUE_LEN); + uint8_t signature[ED25519_SIGNATURE_LEN]; + ASSERT_THAT(ED25519_sign(signature, value_to_sign, sizeof(value_to_sign), + signing_private_key_), + Eq(1)); + signed_encryption_key.set_signature(std::string( + reinterpret_cast<const char*>(signature), ED25519_SIGNATURE_LEN)); + // Double check signature. + ASSERT_THAT(ED25519_verify(value_to_sign, sizeof(value_to_sign), signature, + signature_verification_public_key_), + Eq(1)); + storage->UpdateEncryptionKey(signed_encryption_key); + } + + bool is_encryption_enabled() const { return ::testing::get<0>(GetParam()); } + size_t single_file_size_limit() const { + return ::testing::get<1>(GetParam()); + } + + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + + base::test::ScopedFeatureList scoped_feature_list_; + + uint8_t signature_verification_public_key_[ED25519_PUBLIC_KEY_LEN]; + uint8_t signing_private_key_[ED25519_PRIVATE_KEY_LEN]; + + base::ScopedTempDir location_; + scoped_refptr<Decryptor> decryptor_; + scoped_refptr<Storage> storage_; + bool expect_to_need_key_{false}; + + // Test-wide global mapping of <generation id, sequencing id> to record + // digest. Serves all MockUploadClients created by test fixture. + MockUploadClient::LastRecordDigestMap last_record_digest_map_; + // Guard Access to last_record_digest_map_ + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_{ + base::ThreadPool::CreateSequencedTaskRunner(base::TaskTraits())}; + + ::testing::MockFunction< + void(Priority, bool /*need_encryption_key*/, MockUploadClient*)> + set_mock_uploader_expectations_; +}; + +constexpr std::array<const char*, 3> kData = {"Rec1111", "Rec222", "Rec33"}; +constexpr std::array<const char*, 3> kMoreData = {"More1111", "More222", + "More33"}; + +TEST_P(StorageTest, WriteIntoNewStorageAndReopen) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + EXPECT_CALL(set_mock_uploader_expectations_, Call(_, _, NotNull())).Times(0); + WriteStringOrDie(FAST_BATCH, kData[0]); + WriteStringOrDie(FAST_BATCH, kData[1]); + WriteStringOrDie(FAST_BATCH, kData[2]); + + ResetTestStorage(); + + CreateTestStorageOrDie(BuildTestStorageOptions()); +} + +TEST_P(StorageTest, WriteIntoNewStorageReopenAndWriteMore) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + EXPECT_CALL(set_mock_uploader_expectations_, Call(_, _, NotNull())).Times(0); + WriteStringOrDie(FAST_BATCH, kData[0]); + WriteStringOrDie(FAST_BATCH, kData[1]); + WriteStringOrDie(FAST_BATCH, kData[2]); + + ResetTestStorage(); + + CreateTestStorageOrDie(BuildTestStorageOptions()); + WriteStringOrDie(FAST_BATCH, kMoreData[0]); + WriteStringOrDie(FAST_BATCH, kMoreData[1]); + WriteStringOrDie(FAST_BATCH, kMoreData[2]); +} + +TEST_P(StorageTest, WriteIntoNewStorageAndUpload) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + WriteStringOrDie(FAST_BATCH, kData[0]); + WriteStringOrDie(FAST_BATCH, kData[1]); + WriteStringOrDie(FAST_BATCH, kData[2]); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + }))); + + // Trigger upload. + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageTest, WriteIntoNewStorageAndUploadWithKeyUpdate) { + // Run the test only when encryption is enabled. + if (!is_encryption_enabled()) { + return; + } + + static constexpr auto kKeyRenewalTime = base::TimeDelta::FromSeconds(5); + CreateTestStorageOrDie( + BuildTestStorageOptions(), + base::MakeRefCounted<EncryptionModule>(kKeyRenewalTime)); + WriteStringOrDie(MANUAL_BATCH, kData[0]); + WriteStringOrDie(MANUAL_BATCH, kData[1]); + WriteStringOrDie(MANUAL_BATCH, kData[2]); + + // Set uploader expectations. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Ne(MANUAL_BATCH), /*need_encryption_key=*/_, NotNull())) + .WillRepeatedly(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetEmpty client(mock_upload_client); + }))); + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(MANUAL_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + }))); + + // Trigger upload with no key update. + EXPECT_OK(storage_->Flush(MANUAL_BATCH)); + + // Write more data. + WriteStringOrDie(MANUAL_BATCH, kMoreData[0]); + WriteStringOrDie(MANUAL_BATCH, kMoreData[1]); + WriteStringOrDie(MANUAL_BATCH, kMoreData[2]); + + // Wait to trigger encryption key request on the next upload + task_environment_.FastForwardBy(kKeyRenewalTime + + base::TimeDelta::FromSeconds(1)); + + // Set uploader expectations with encryption key request. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(MANUAL_BATCH), /*need_encryption_key=*/Eq(true), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + }))); + + // Trigger upload with key update after a long wait. + EXPECT_OK(storage_->Flush(MANUAL_BATCH)); +} + +TEST_P(StorageTest, WriteIntoNewStorageReopenWriteMoreAndUpload) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + WriteStringOrDie(FAST_BATCH, kData[0]); + WriteStringOrDie(FAST_BATCH, kData[1]); + WriteStringOrDie(FAST_BATCH, kData[2]); + + ResetTestStorage(); + + CreateTestStorageOrDie(BuildTestStorageOptions()); + WriteStringOrDie(FAST_BATCH, kMoreData[0]); + WriteStringOrDie(FAST_BATCH, kMoreData[1]); + WriteStringOrDie(FAST_BATCH, kMoreData[2]); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + }))); + + // Trigger upload. + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageTest, WriteIntoNewStorageAndFlush) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + WriteStringOrDie(MANUAL_BATCH, kData[0]); + WriteStringOrDie(MANUAL_BATCH, kData[1]); + WriteStringOrDie(MANUAL_BATCH, kData[2]); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(MANUAL_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + }))); + + // Trigger upload. + EXPECT_OK(storage_->Flush(MANUAL_BATCH)); +} + +TEST_P(StorageTest, WriteIntoNewStorageReopenWriteMoreAndFlush) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + WriteStringOrDie(MANUAL_BATCH, kData[0]); + WriteStringOrDie(MANUAL_BATCH, kData[1]); + WriteStringOrDie(MANUAL_BATCH, kData[2]); + + ResetTestStorage(); + + CreateTestStorageOrDie(BuildTestStorageOptions()); + WriteStringOrDie(MANUAL_BATCH, kMoreData[0]); + WriteStringOrDie(MANUAL_BATCH, kMoreData[1]); + WriteStringOrDie(MANUAL_BATCH, kMoreData[2]); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(MANUAL_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + }))); + + // Trigger upload. + EXPECT_OK(storage_->Flush(MANUAL_BATCH)); +} + +TEST_P(StorageTest, WriteAndRepeatedlyUploadWithConfirmations) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + + WriteStringOrDie(FAST_BATCH, kData[0]); + WriteStringOrDie(FAST_BATCH, kData[1]); + WriteStringOrDie(FAST_BATCH, kData[2]); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + }))); + + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #0 and forward time again, removing data #0 + ConfirmOrDie(FAST_BATCH, /*sequencing_id=*/0); + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(1, kData[1]) + .Required(2, kData[2]); + }))); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #1 and forward time again, removing data #1 + ConfirmOrDie(FAST_BATCH, /*sequencing_id=*/1); + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(2, kData[2]); + }))); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Add more records and verify that #2 and new records are returned. + WriteStringOrDie(FAST_BATCH, kMoreData[0]); + WriteStringOrDie(FAST_BATCH, kMoreData[1]); + WriteStringOrDie(FAST_BATCH, kMoreData[2]); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + }))); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #2 and forward time again, removing data #2 + ConfirmOrDie(FAST_BATCH, /*sequencing_id=*/2); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + }))); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +TEST_P(StorageTest, WriteAndRepeatedlyImmediateUpload) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + + // Upload is initiated asynchronously, so it may happen after the next + // record is also written. Because of that we set expectations for the + // records after the current one as |Possible|. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, + kData[0]); // Immediately uploads and verifies. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Possible(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, + kData[1]); // Immediately uploads and verifies. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, + kData[2]); // Immediately uploads and verifies. +} + +TEST_P(StorageTest, WriteAndRepeatedlyImmediateUploadWithConfirmations) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + + // Upload is initiated asynchronously, so it may happen after the next + // record is also written. Because of the Confirmation below, we set + // expectations for the records that may be eliminated by Confirmation as + // |Possible|. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kData[0]); + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kData[1]); + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Required(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kData[2]); + + // Confirm #1, removing data #0 and #1 + ConfirmOrDie(IMMEDIATE, /*sequencing_id=*/1); + + // Add more records and verify that #2 and new records are returned. + // Upload is initiated asynchronously, so it may happen after the next + // record is also written. Because of that we set expectations for the + // records after the current one as |Possible|. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Possible(4, kMoreData[1]) + .Possible(5, kMoreData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kMoreData[0]); + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Possible(5, kMoreData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kMoreData[1]); + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(2, kData[2]) + .Required(3, kMoreData[0]) + .Required(4, kMoreData[1]) + .Required(5, kMoreData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kMoreData[2]); +} + +TEST_P(StorageTest, WriteAndRepeatedlyUploadMultipleQueues) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + + // Upload is initiated asynchronously, so it may happen after the next + // record is also written. Because of the Confirmation below, we set + // expectations for the records that may be eliminated by Confirmation as + // |Possible|. + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kData[0]); + WriteStringOrDie(SLOW_BATCH, kMoreData[0]); + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Possible(0, kData[0]) + .Possible(1, kData[1]) + .Possible(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kData[1]); + WriteStringOrDie(SLOW_BATCH, kMoreData[1]); + + // Set uploader expectations for SLOW_BATCH. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillRepeatedly(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetEmpty client(mock_upload_client); + }))); + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(SLOW_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Required(0, kMoreData[0]) + .Required(1, kMoreData[1]); + }))); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(20)); + + // Confirm #0 SLOW_BATCH, removing data #0 + ConfirmOrDie(SLOW_BATCH, /*sequencing_id=*/0); + + // Confirm #1 IMMEDIATE, removing data #0 and #1 + ConfirmOrDie(IMMEDIATE, /*sequencing_id=*/1); + + // Add more data + EXPECT_CALL(set_mock_uploader_expectations_, + Call(Eq(IMMEDIATE), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(priority, mock_upload_client) + .Possible(1, kData[1]) + .Required(2, kData[2]); + }))); + WriteStringOrDie(IMMEDIATE, kData[2]); + WriteStringOrDie(SLOW_BATCH, kMoreData[2]); + + // Set uploader expectations for SLOW_BATCH. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillRepeatedly(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetEmpty client(mock_upload_client); + }))); + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(SLOW_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(SLOW_BATCH, mock_upload_client) + .Required(1, kMoreData[1]) + .Required(2, kMoreData[2]); + }))); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(20)); +} + +TEST_P(StorageTest, WriteEncryptFailure) { + auto test_encryption_module = + base::MakeRefCounted<test::TestEncryptionModule>(); + CreateTestStorageOrDie(BuildTestStorageOptions(), test_encryption_module); + EXPECT_CALL(*test_encryption_module, EncryptRecord(_, _)) + .WillOnce(WithArg<1>( + Invoke([](base::OnceCallback<void(StatusOr<EncryptedRecord>)> cb) { + std::move(cb).Run(Status(error::UNKNOWN, "Failing for tests")); + }))); + const Status result = WriteString(FAST_BATCH, "TEST_MESSAGE"); + EXPECT_FALSE(result.ok()); + EXPECT_EQ(result.error_code(), error::UNKNOWN); +} + +TEST_P(StorageTest, ForceConfirm) { + CreateTestStorageOrDie(BuildTestStorageOptions()); + + WriteStringOrDie(FAST_BATCH, kData[0]); + WriteStringOrDie(FAST_BATCH, kData[1]); + WriteStringOrDie(FAST_BATCH, kData[2]); + + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(FAST_BATCH, mock_upload_client) + .Required(0, kData[0]) + .Required(1, kData[1]) + .Required(2, kData[2]); + }))); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Confirm #1 and forward time again, possibly removing records #0 and #1 + ConfirmOrDie(FAST_BATCH, /*sequencing_id=*/1); + // Set uploader expectations. + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(FAST_BATCH, mock_upload_client) + .Required(2, kData[2]); + }))); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Now force confirm #0 and forward time again. + ConfirmOrDie(FAST_BATCH, /*sequencing_id=*/base::nullopt, /*force=*/true); + // Set uploader expectations: #0 and #1 could be returned as Gaps + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(FAST_BATCH, mock_upload_client) + .RequiredSeqId(0) + .RequiredSeqId(1) + .RequiredSeqId(2) + // 0-2 must have been encountered, but actual contents + // can be different: + .Possible(0, kData[0]) + .PossibleGap(0, 1) + .PossibleGap(0, 2) + .Possible(1, kData[1]) + .Required(2, kData[2]); + }))); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); + + // Force confirm #0 and forward time again. + ConfirmOrDie(FAST_BATCH, /*sequencing_id=*/0, /*force=*/true); + // Set uploader expectations: #0 and #1 could be returned as Gaps + EXPECT_CALL( + set_mock_uploader_expectations_, + Call(Eq(FAST_BATCH), /*need_encryption_key=*/Eq(false), NotNull())) + .WillOnce(WithArgs<0, 2>( + Invoke([](Priority priority, MockUploadClient* mock_upload_client) { + MockUploadClient::SetUp(FAST_BATCH, mock_upload_client) + .RequiredSeqId(1) + .RequiredSeqId(2) + // 0-2 must have been encountered, but actual contents + // can be different: + .PossibleGap(1, 1) + .Possible(1, kData[1]) + .Required(2, kData[2]); + }))); + // Forward time to trigger upload + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(1)); +} + +INSTANTIATE_TEST_SUITE_P( + VaryingFileSize, + StorageTest, + ::testing::Combine(::testing::Bool() /* true - encryption enabled */, + ::testing::Values(128 * 1024LL * 1024LL, + 256 /* two records in file */, + 1 /* single record in file */))); + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_uploader_interface.cc b/chromium/components/reporting/storage/storage_uploader_interface.cc new file mode 100644 index 00000000000..8eb0841f497 --- /dev/null +++ b/chromium/components/reporting/storage/storage_uploader_interface.cc @@ -0,0 +1,12 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/storage_uploader_interface.h" + +namespace reporting { + +UploaderInterface::UploaderInterface() = default; +UploaderInterface::~UploaderInterface() = default; + +} // namespace reporting diff --git a/chromium/components/reporting/storage/storage_uploader_interface.h b/chromium/components/reporting/storage/storage_uploader_interface.h new file mode 100644 index 00000000000..037fc4c6095 --- /dev/null +++ b/chromium/components/reporting/storage/storage_uploader_interface.h @@ -0,0 +1,64 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_STORAGE_UPLOADER_INTERFACE_H_ +#define COMPONENTS_REPORTING_STORAGE_STORAGE_UPLOADER_INTERFACE_H_ + +#include <cstdint> +#include <memory> + +#include "base/callback.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// Interface for Upload by StorageModule. +// Must be implemented by an object returned by |StartUpload| callback (see +// below). Every time on of the StorageQueue's starts an upload (by timer or +// immediately after Write) it uses this interface to hand available records +// over to the actual uploader. StorageQueue takes ownership of it and +// automatically discards after |Completed| returns. +class UploaderInterface { + public: + // Callback type for UploadInterface provider for specified priority. + // |priority| identifies which queue is going to upload the data. + // Set |need_encryption_key| if key is needed (initially or periodically). + using StartCb = + base::RepeatingCallback<StatusOr<std::unique_ptr<UploaderInterface>>( + Priority priority, + bool need_encryption_key)>; + + UploaderInterface(const UploaderInterface& other) = delete; + const UploaderInterface& operator=(const UploaderInterface& other) = delete; + virtual ~UploaderInterface(); + + // Unserializes every record and hands ownership over for processing (e.g. + // to add to the network message). Expects |processed_cb| to be called after + // the record or error status has been processed, with true if next record + // needs to be delivered and false if the Uploader should stop. + virtual void ProcessRecord(EncryptedRecord record, + base::OnceCallback<void(bool)> processed_cb) = 0; + + // Makes a note of a gap [start, start + count). Expects |processed_cb| to + // be called after the record or error status has been processed, with true + // if next record needs to be delivered and false if the Uploader should + // stop. + virtual void ProcessGap(SequencingInformation start, + uint64_t count, + base::OnceCallback<void(bool)> processed_cb) = 0; + + // Finalizes the upload (e.g. sends the message to server and gets + // response). Called always, regardless of whether there were errors. + virtual void Completed(Status final_status) = 0; + + protected: + UploaderInterface(); +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_STORAGE_UPLOADER_INTERFACE_H_ diff --git a/chromium/components/reporting/storage/test_storage_module.cc b/chromium/components/reporting/storage/test_storage_module.cc new file mode 100644 index 00000000000..bbdd49709a6 --- /dev/null +++ b/chromium/components/reporting/storage/test_storage_module.cc @@ -0,0 +1,48 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/storage/test_storage_module.h" + +#include <utility> + +#include "base/callback.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/storage/storage_module_interface.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::Invoke; + +namespace reporting { +namespace test { + +TestStorageModuleStrict::TestStorageModuleStrict() { + ON_CALL(*this, AddRecord) + .WillByDefault(Invoke(this, &TestStorageModule::AddRecordSuccessfully)); +} + +TestStorageModuleStrict::~TestStorageModuleStrict() = default; + +Record TestStorageModuleStrict::record() const { + EXPECT_TRUE(record_.has_value()); + return record_.value(); +} + +Priority TestStorageModuleStrict::priority() const { + EXPECT_TRUE(priority_.has_value()); + return priority_.value(); +} + +void TestStorageModuleStrict::AddRecordSuccessfully( + Priority priority, + Record record, + base::OnceCallback<void(Status)> callback) { + record_ = std::move(record); + priority_ = priority; + std::move(callback).Run(Status::StatusOK()); +} + +} // namespace test +} // namespace reporting diff --git a/chromium/components/reporting/storage/test_storage_module.h b/chromium/components/reporting/storage/test_storage_module.h new file mode 100644 index 00000000000..26d11b03600 --- /dev/null +++ b/chromium/components/reporting/storage/test_storage_module.h @@ -0,0 +1,65 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_STORAGE_TEST_STORAGE_MODULE_H_ +#define COMPONENTS_REPORTING_STORAGE_TEST_STORAGE_MODULE_H_ + +#include <utility> + +#include "base/callback.h" +#include "base/optional.h" +#include "components/reporting/proto/record.pb.h" +#include "components/reporting/proto/record_constants.pb.h" +#include "components/reporting/storage/storage_module_interface.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace reporting { +namespace test { + +class TestStorageModuleStrict : public StorageModuleInterface { + public: + // As opposed to the production |StorageModule|, test module does not need to + // call factory method - it is created directly by constructor. + TestStorageModuleStrict(); + + MOCK_METHOD(void, + AddRecord, + (Priority priority, + Record record, + base::OnceCallback<void(Status)> callback), + (override)); + + MOCK_METHOD(void, + ReportSuccess, + (SequencingInformation sequencing_information, bool force), + (override)); + + MOCK_METHOD(void, + UpdateEncryptionKey, + (SignedEncryptionInfo signed_encryption_key), + (override)); + + Record record() const; + Priority priority() const; + + protected: + ~TestStorageModuleStrict() override; + + private: + void AddRecordSuccessfully(Priority priority, + Record record, + base::OnceCallback<void(Status)> callback); + + base::Optional<Record> record_; + base::Optional<Priority> priority_; +}; + +// Most of the time no need to log uninterested calls to |AddRecord|. +typedef ::testing::NiceMock<TestStorageModuleStrict> TestStorageModule; + +} // namespace test +} // namespace reporting + +#endif // COMPONENTS_REPORTING_STORAGE_TEST_STORAGE_MODULE_H_ diff --git a/chromium/components/reporting/util/BUILD.gn b/chromium/components/reporting/util/BUILD.gn new file mode 100644 index 00000000000..421347fd64c --- /dev/null +++ b/chromium/components/reporting/util/BUILD.gn @@ -0,0 +1,92 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. Use +# of this source code is governed by a BSD-style license that can +# be found in the LICENSE file. + +import("//build/config/features.gni") +import("//third_party/libprotobuf-mutator/fuzzable_proto_library.gni") +import("//third_party/protobuf/proto_library.gni") + +static_library("backoff_settings") { + sources = [ + "backoff_settings.cc", + "backoff_settings.h", + ] + + deps = [ "//net" ] +} + +source_set("shared_vector") { + sources = [ "shared_vector.h" ] + deps = [ + ":status", + "//base", + ] +} + +proto_library("status_proto") { + visibility = [ + "//chrome/browser:browser", + "//chrome/browser:test_support", + "//components/reporting/*", + ] + + sources = [ "status.proto" ] + + proto_out_dir = "components/reporting/util" +} + +static_library("status") { + sources = [ + "status.cc", + "status.h", + "statusor.cc", + "statusor.h", + ] + public_deps = [ ":status_proto" ] + deps = [ "//base" ] +} + +source_set("shared_queue") { + sources = [ "shared_queue.h" ] + deps = [ + ":status", + "//base", + ] +} + +source_set("status_macros") { + sources = [ "status_macros.h" ] + + deps = [ ":status" ] +} + +source_set("task_runner_context") { + sources = [ "task_runner_context.h" ] + + deps = [ "//base" ] +} + +# All unit tests are built as part of the //components:components_unittests +# target. +source_set("unit_tests") { + testonly = true + sources = [ + "shared_queue_unittest.cc", + "shared_vector_unittest.cc", + "status_macros_unittest.cc", + "status_unittest.cc", + "statusor_unittest.cc", + ] + deps = [ + ":shared_queue", + ":shared_vector", + ":status", + ":status_macros", + ":status_proto", + ":task_runner_context", + "//base", + "//base/test:test_support", + "//testing/gmock", + "//testing/gtest", + ] +} diff --git a/chromium/components/reporting/util/DEPS b/chromium/components/reporting/util/DEPS new file mode 100644 index 00000000000..68bee72b387 --- /dev/null +++ b/chromium/components/reporting/util/DEPS @@ -0,0 +1,4 @@ +include_rules = [ + "+base", + "+net/base/backoff_entry.h", +] diff --git a/chromium/components/reporting/util/backoff_settings.cc b/chromium/components/reporting/util/backoff_settings.cc new file mode 100644 index 00000000000..adbd62bd383 --- /dev/null +++ b/chromium/components/reporting/util/backoff_settings.cc @@ -0,0 +1,40 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/backoff_settings.h" + +#include <memory> + +#include "net/base/backoff_entry.h" + +namespace reporting { + +std::unique_ptr<::net::BackoffEntry> GetBackoffEntry() { + // Retry starts with 10 second delay and is doubled with every failure. + static const net::BackoffEntry::Policy kDefaultUploadBackoffPolicy = { + // Number of initial errors to ignore before applying + // exponential back-off rules. + /*num_errors_to_ignore=*/0, + + // Initial delay is 10 seconds. + /*initial_delay_ms=*/10 * 1000, + + // Factor by which the waiting time will be multiplied. + /*multiply_factor=*/2, + + // Fuzzing percentage. + /*jitter_factor=*/0.1, + + // Maximum delay is 90 seconds. + /*maximum_backoff_ms=*/90 * 1000, + + // It's up to the caller to reset the backoff time. + /*entry_lifetime_ms=*/-1, + + /*always_use_initial_delay=*/true, + }; + return std::make_unique<::net::BackoffEntry>(&kDefaultUploadBackoffPolicy); +} + +} // namespace reporting diff --git a/chromium/components/reporting/util/backoff_settings.h b/chromium/components/reporting/util/backoff_settings.h new file mode 100644 index 00000000000..5facb24f992 --- /dev/null +++ b/chromium/components/reporting/util/backoff_settings.h @@ -0,0 +1,22 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_UTIL_BACKOFF_SETTINGS_H_ +#define COMPONENTS_REPORTING_UTIL_BACKOFF_SETTINGS_H_ + +#include <memory> + +#include "net/base/backoff_entry.h" + +namespace reporting { + +// Returns a BackoffEntry object that defaults to initial 10 second delay and +// doubles the delay on every failure, to a maximum delay of 90 seconds. +// Caller owns the object and is responsible for resetting the delay on +// successful completion. +std::unique_ptr<::net::BackoffEntry> GetBackoffEntry(); + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_UTIL_BACKOFF_SETTINGS_H_ diff --git a/chromium/components/reporting/util/shared_queue.h b/chromium/components/reporting/util/shared_queue.h new file mode 100644 index 00000000000..923b2e7aa33 --- /dev/null +++ b/chromium/components/reporting/util/shared_queue.h @@ -0,0 +1,99 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_UTIL_SHARED_QUEUE_H_ +#define COMPONENTS_REPORTING_UTIL_SHARED_QUEUE_H_ + +#include <utility> + +#include "base/containers/queue.h" +#include "base/memory/ref_counted.h" +#include "base/sequenced_task_runner.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// SharedQueue wraps a |base::queue| and ensures access happens on a +// SequencedTaskRunner. +template <typename QueueType> +class SharedQueue : public base::RefCountedThreadSafe<SharedQueue<QueueType>> { + public: + static scoped_refptr<SharedQueue<QueueType>> Create() { + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner{ + base::ThreadPool::CreateSequencedTaskRunner({})}; + return base::WrapRefCounted( + new SharedQueue<QueueType>(sequenced_task_runner)); + } + + // Push will schedule a push of |item| onto the queue and call + // |push_complete_cb| once complete. + void Push(QueueType item, base::OnceCallback<void()> push_complete_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, base::BindOnce(&SharedQueue::OnPush, this, std::move(item), + std::move(push_complete_cb))); + } + + // Pop will schedule a pop off the queue and call |get_pop_cb| once complete. + // If the queue is empty, |get_pop_cb| will be called with + // error::OUT_OF_RANGE. + void Pop(base::OnceCallback<void(StatusOr<QueueType>)> get_pop_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&SharedQueue::OnPop, this, std::move(get_pop_cb))); + } + + // Swap will schedule a swap of the |queue_| contents with the provided + // |new_queue|, and send the old contents to the |swap_queue_cb|. + void Swap(base::queue<QueueType> new_queue, + base::OnceCallback<void(base::queue<QueueType>)> swap_queue_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&SharedQueue::OnSwap, this, std::move(new_queue), + std::move(swap_queue_cb))); + } + + protected: + virtual ~SharedQueue() = default; + + private: + friend class base::RefCountedThreadSafe<SharedQueue<QueueType>>; + + explicit SharedQueue( + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner) + : sequenced_task_runner_(sequenced_task_runner) {} + + void OnPush(QueueType item, base::OnceCallback<void()> push_complete_cb) { + queue_.push(std::move(item)); + std::move(push_complete_cb).Run(); + } + + void OnPop(base::OnceCallback<void(StatusOr<QueueType>)> cb) { + if (queue_.empty()) { + std::move(cb).Run(Status(error::OUT_OF_RANGE, "Queue is empty")); + return; + } + + QueueType item = std::move(queue_.front()); + queue_.pop(); + std::move(cb).Run(std::move(item)); + } + + void OnSwap(base::queue<QueueType> new_queue, + base::OnceCallback<void(base::queue<QueueType>)> swap_queue_cb) { + queue_.swap(new_queue); + std::move(swap_queue_cb).Run(std::move(new_queue)); + } + + // Used to monitor if the callback is in use or not. + base::queue<QueueType> queue_; + + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_UTIL_SHARED_QUEUE_H_ diff --git a/chromium/components/reporting/util/shared_queue_unittest.cc b/chromium/components/reporting/util/shared_queue_unittest.cc new file mode 100644 index 00000000000..b0a3a3de86f --- /dev/null +++ b/chromium/components/reporting/util/shared_queue_unittest.cc @@ -0,0 +1,159 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/shared_queue.h" + +#include "base/callback_helpers.h" +#include "base/sequenced_task_runner.h" +#include "base/synchronization/waitable_event.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" +#include "base/test/task_environment.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +#include "testing/gtest/include/gtest/gtest.h" + +namespace reporting { +namespace { + +class QueueTester { + public: + explicit QueueTester(scoped_refptr<SharedQueue<int>> queue) + : queue_(queue), + sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner({})), + completed_(base::WaitableEvent::ResetPolicy::MANUAL, + base::WaitableEvent::InitialState::NOT_SIGNALED), + pop_result_(Status(error::FAILED_PRECONDITION, "Pop hasn't run yet")) {} + + ~QueueTester() = default; + + void Push(int value) { queue_->Push(value, base::DoNothing()); } + + void Pop() { + sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&QueueTester::PopInternal, base::Unretained(this))); + } + + void Swap() { + queue_->Swap(base::queue<int>(), + base::BindOnce(&QueueTester::OnSwap, base::Unretained(this))); + } + + void PushPop(int value) { + queue_->Push(value, base::BindOnce(&QueueTester::OnPushPopComplete, + base::Unretained(this))); + } + + void Wait() { + completed_.Wait(); + completed_.Reset(); + } + + StatusOr<int> pop_result() { return pop_result_; } + base::queue<int>* swap_result() { return &swap_result_; } + + private: + void OnPushPopComplete() { Pop(); } + + void PopInternal() { + queue_->Pop( + base::BindOnce(&QueueTester::OnPopComplete, base::Unretained(this))); + } + + void OnPopComplete(StatusOr<int> pop_result) { + pop_result_ = pop_result; + Signal(); + } + + void OnSwap(base::queue<int> swap_result) { + swap_result_ = swap_result; + Signal(); + } + + void Signal() { completed_.Signal(); } + + scoped_refptr<SharedQueue<int>> queue_; + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_; + base::WaitableEvent completed_; + + StatusOr<int> pop_result_; + base::queue<int> swap_result_; +}; + +TEST(SharedQueueTest, SuccessfulPushPop) { + base::test::TaskEnvironment task_envrionment{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + + const int kExpectedValue = 1234; + + auto queue = SharedQueue<int>::Create(); + QueueTester queue_tester(queue); + + queue_tester.PushPop(kExpectedValue); + queue_tester.Wait(); + + auto pop_result = queue_tester.pop_result(); + ASSERT_OK(pop_result); + EXPECT_EQ(pop_result.ValueOrDie(), kExpectedValue); +} + +TEST(SharedQueueTest, PushOrderMaintained) { + base::test::TaskEnvironment task_envrionment{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + + std::vector<int> kExpectedValues = {1, 1, 2, 3, 5, 8, 13, 21}; + + auto queue = SharedQueue<int>::Create(); + QueueTester queue_tester(queue); + + for (auto value : kExpectedValues) { + queue_tester.Push(value); + } + + for (auto value : kExpectedValues) { + queue_tester.Pop(); + queue_tester.Wait(); + auto pop_result = queue_tester.pop_result(); + ASSERT_OK(pop_result); + EXPECT_EQ(pop_result.ValueOrDie(), value); + } +} + +TEST(SharedQueueTest, SwapSuccessful) { + base::test::TaskEnvironment task_envrionment{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + + std::vector<int> kExpectedValues = {1, 1, 2, 3, 5, 8, 13, 21}; + + auto queue = SharedQueue<int>::Create(); + QueueTester queue_tester(queue); + + for (auto value : kExpectedValues) { + queue_tester.Push(value); + } + + queue_tester.Swap(); + queue_tester.Wait(); + + auto* swapped_queue = queue_tester.swap_result(); + + for (auto value : kExpectedValues) { + EXPECT_EQ(swapped_queue->front(), value); + swapped_queue->pop(); + } + + // Test to ensure the SharedQueue is empty. + queue_tester.Pop(); + queue_tester.Wait(); + + auto pop_result = queue_tester.pop_result(); + + EXPECT_FALSE(pop_result.ok()); + EXPECT_EQ(pop_result.status().error_code(), error::OUT_OF_RANGE); +} + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/util/shared_vector.h b/chromium/components/reporting/util/shared_vector.h new file mode 100644 index 00000000000..9f865098084 --- /dev/null +++ b/chromium/components/reporting/util/shared_vector.h @@ -0,0 +1,199 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_UTIL_SHARED_VECTOR_H_ +#define COMPONENTS_REPORTING_UTIL_SHARED_VECTOR_H_ + +#include <utility> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/callback_helpers.h" +#include "base/containers/queue.h" +#include "base/memory/ref_counted.h" +#include "base/sequence_checker.h" +#include "base/sequenced_task_runner.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// SharedVector wraps a |std::vector| and ensures access happens on a +// SequencedTaskRunner. +template <typename VectorType> +class SharedVector + : public base::RefCountedThreadSafe<SharedVector<VectorType>> { + public: + static scoped_refptr<SharedVector<VectorType>> Create() { + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner{ + base::ThreadPool::CreateSequencedTaskRunner({})}; + return base::WrapRefCounted( + new SharedVector<VectorType>(sequenced_task_runner)); + } + + void PushBack(VectorType item, + base::OnceCallback<void()> push_back_complete_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&SharedVector::OnPushBack, this, std::move(item), + std::move(push_back_complete_cb))); + } + + // Erase will call erase on all elements that return true for the + // |predicate_cb|. + void Erase(base::RepeatingCallback<bool(const VectorType&)> predicate_cb, + base::OnceCallback<void(size_t)> remove_complete_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&SharedVector::OnErase, this, std::move(predicate_cb), + std::move(remove_complete_cb))); + } + + // Provided as the nearest equivalent to std::vector::find. A regular find + // operation may be invalid by the time a caller is notified of its existence. + // |predicate_cb| is called on each element. If |predicate_cb| returns true + // |found_cb| is called on the same element, ending the search. + // |not_found_cb| is called if no elements return true. + void ExecuteIfFound( + base::RepeatingCallback<bool(const VectorType&)> predicate_cb, + base::OnceCallback<void(VectorType&)> found_cb, + base::OnceCallback<void()> not_found_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, base::BindOnce(&SharedVector::OnExecuteIfFound, this, + std::move(predicate_cb), std::move(found_cb), + std::move(not_found_cb))); + } + + // Iterates over each element in |vector_|, and calls |predicate_cb|. If + // |predicate_cb| returns true, |element_executor| will be called on the same + // element and iteration will continue. At the end of iteration + // |execute_complete_cb| will be called. + // A default |predicate_cb| is provided that always returns true. + void ExecuteOnEachElement( + base::RepeatingCallback<void(VectorType&)> element_executor, + base::OnceCallback<void()> execute_complete_cb, + base::RepeatingCallback<bool(const VectorType&)> predicate_cb = + base::BindRepeating([](const VectorType&) { return true; })) { + sequenced_task_runner_->PostTask( + FROM_HERE, base::BindOnce(&SharedVector::OnExecuteOnEachElement, this, + std::move(element_executor), + std::move(execute_complete_cb), + std::move(predicate_cb))); + } + + void IsEmpty(base::OnceCallback<void(bool)> get_empty_cb) { + sequenced_task_runner_->PostTask( + FROM_HERE, base::BindOnce(&SharedVector::OnIsEmpty, this, + std::move(get_empty_cb))); + } + + protected: + virtual ~SharedVector() = default; + + private: + friend class base::RefCountedThreadSafe<SharedVector<VectorType>>; + + explicit SharedVector( + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner) + : sequenced_task_runner_(sequenced_task_runner) { + DETACH_FROM_SEQUENCE(sequence_checker_); + } + + void OnPushBack(VectorType item, + base::OnceCallback<void()> push_back_complete_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + vector_.push_back(std::move(item)); + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](base::OnceCallback<void()> push_back_complete_cb) { + std::move(push_back_complete_cb).Run(); + }, + std::move(push_back_complete_cb))); + } + + void OnErase(base::RepeatingCallback<bool(const VectorType&)> predicate_cb, + base::OnceCallback<void(size_t)> remove_complete_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + size_t number_erased = 0; + for (auto it = vector_.begin(); it != vector_.end();) { + if (predicate_cb.Run(*it)) { + it = vector_.erase(it); + number_erased++; + } else { + it++; + } + } + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](base::OnceCallback<void(size_t)> remove_complete_cb, + size_t number_erased) { + std::move(remove_complete_cb).Run(number_erased); + }, + std::move(remove_complete_cb), number_erased)); + } + + void OnExecuteIfFound( + base::RepeatingCallback<bool(const VectorType&)> predicate_cb, + base::OnceCallback<void(VectorType&)> found_cb, + base::OnceCallback<void()> not_found_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + for (VectorType& element : vector_) { + if (predicate_cb.Run(element)) { + std::move(found_cb).Run(element); + return; + } + } + base::ThreadPool::PostTask(FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](base::OnceCallback<void()> not_found_cb) { + std::move(not_found_cb).Run(); + }, + std::move(not_found_cb))); + } + + void OnExecuteOnEachElement( + base::RepeatingCallback<void(VectorType&)> element_executor, + base::OnceCallback<void()> execute_complete_cb, + base::RepeatingCallback<bool(const VectorType&)> predicate_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + for (VectorType& element : vector_) { + if (predicate_cb.Run(element)) { + element_executor.Run(element); + } else { + break; + } + } + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](base::OnceCallback<void()> execute_complete_cb) { + std::move(execute_complete_cb).Run(); + }, + std::move(execute_complete_cb))); + } + + void OnIsEmpty(base::OnceCallback<void(bool)> is_empty_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT}, + base::BindOnce( + [](base::OnceCallback<void(bool)> is_empty_cb, bool is_empty) { + std::move(is_empty_cb).Run(is_empty); + }, + std::move(is_empty_cb), vector_.empty())); + } + + std::vector<VectorType> vector_; + + SEQUENCE_CHECKER(sequence_checker_); + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_UTIL_SHARED_VECTOR_H_ diff --git a/chromium/components/reporting/util/shared_vector_unittest.cc b/chromium/components/reporting/util/shared_vector_unittest.cc new file mode 100644 index 00000000000..0e56e14302d --- /dev/null +++ b/chromium/components/reporting/util/shared_vector_unittest.cc @@ -0,0 +1,334 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/shared_vector.h" + +#include "base/bind.h" +#include "base/callback.h" +#include "base/callback_helpers.h" +#include "base/optional.h" +#include "base/sequenced_task_runner.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" +#include "base/test/task_environment.h" + +#include "testing/gtest/include/gtest/gtest.h" + +namespace reporting { +namespace { + +template <typename VectorType> +class VectorTester { + public: + // FindType must be copyable. + template <typename FindType> + class Finder { + public: + explicit Finder(const FindType& item) + : sought_item_(item), run_loop_(std::make_unique<base::RunLoop>()) {} + + bool Compare(const FindType& item) const { return sought_item_ == item; } + + void OnFound(FindType& item) { + found_result_ = item; + run_loop_->Quit(); + } + + void OnNotFound() { run_loop_->Quit(); } + + const FindType& sought_item() const { return sought_item_; } + + const base::Optional<FindType>& found_result() const { + return found_result_; + } + + void Wait() { + run_loop_->Run(); + run_loop_ = std::make_unique<base::RunLoop>(); + } + + private: + const FindType sought_item_; + std::unique_ptr<base::RunLoop> run_loop_; + + base::Optional<FindType> found_result_; + }; + + template <typename ExecuteType> + class Executor { + public: + explicit Executor(size_t expected_value_count) + : expected_value_count_(expected_value_count), + run_loop_(std::make_unique<base::RunLoop>()) {} + + void CountValue(ExecuteType& item) { found_count_++; } + + void Complete() { run_loop_->Quit(); } + + void Wait() { + run_loop_->Run(); + run_loop_ = std::make_unique<base::RunLoop>(); + } + + size_t DifferenceInCount() const { + return expected_value_count_ - found_count_; + } + + size_t found_count() const { return found_count_; } + + private: + const size_t expected_value_count_; + std::unique_ptr<base::RunLoop> run_loop_; + size_t found_count_{0}; + }; + + VectorTester() + : vector_(SharedVector<VectorType>::Create()), + sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner({})), + run_loop_(std::make_unique<base::RunLoop>()) {} + + ~VectorTester() = default; + + // Find only works on copyable items - so VectorType must be copyable. + Finder<VectorType> GetFinder(VectorType sought_item) { + return Finder<VectorType>(sought_item); + } + + Executor<VectorType> GetExecutor(size_t expected_value_count) { + return Executor<VectorType>(expected_value_count); + } + + void PushBack(VectorType item) { + vector_->PushBack( + std::move(item), + base::BindOnce(&VectorTester<VectorType>::OnPushBackComplete, + base::Unretained(this))); + } + + // Resets |insert_success| before returning its value. + base::Optional<bool> GetPushBackSuccess() { + base::Optional<bool> return_value; + return_value.swap(insert_success_); + return return_value; + } + + void Erase(VectorType value) { + auto predicate_cb = base::BindRepeating( + [](const VectorType& expected_value, const VectorType& comparison_value) + -> bool { return expected_value == comparison_value; }, + value); + vector_->Erase(std::move(predicate_cb), + base::BindOnce(&VectorTester<VectorType>::OnEraseComplete, + base::Unretained(this))); + } + + void Erase(base::RepeatingCallback<bool(const VectorType&)> predicate_cb) { + vector_->Erase(std::move(predicate_cb), + base::BindOnce(&VectorTester<VectorType>::OnEraseComplete, + base::Unretained(this))); + } + + base::Optional<uint64_t> GetEraseValue() { + base::Optional<uint64_t> return_value; + return_value.swap(number_deleted_); + return return_value; + } + + void ExecuteIfFound(Finder<VectorType>* finder) { + vector_->ExecuteIfFound( + base::BindRepeating(&Finder<VectorType>::Compare, + base::Unretained(finder)), + base::BindOnce(&Finder<VectorType>::OnFound, base::Unretained(finder)), + base::BindOnce(&Finder<VectorType>::OnNotFound, + base::Unretained(finder))); + } + + void ExecuteOnEachElement(Executor<VectorType>* executor) { + vector_->ExecuteOnEachElement( + base::BindRepeating(&Executor<VectorType>::CountValue, + base::Unretained(executor)), + base::BindOnce(&Executor<VectorType>::Complete, + base::Unretained(executor))); + } + + void Wait() { + run_loop_->Run(); + run_loop_ = std::make_unique<base::RunLoop>(); + } + + private: + void OnPushBackComplete() { + sequenced_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&VectorTester<VectorType>::VectorPushBackSuccess, + base::Unretained(this))); + } + + void VectorPushBackSuccess() { + insert_success_ = true; + Signal(); + } + + void OnEraseComplete(size_t number_deleted) { + sequenced_task_runner_->PostTask( + FROM_HERE, base::BindOnce(&VectorTester<VectorType>::VectorEraseValue, + base::Unretained(this), number_deleted)); + } + + void VectorEraseValue(uint64_t number_deleted) { + number_deleted_ = number_deleted; + Signal(); + } + + void Signal() { run_loop_->Quit(); } + + scoped_refptr<SharedVector<VectorType>> vector_; + scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_; + std::unique_ptr<base::RunLoop> run_loop_; + + base::Optional<bool> insert_success_; + base::Optional<uint64_t> number_deleted_; +}; + +// Ensures that the vector accept values, and will erase inserted values. +TEST(SharedVectorTest, PushBackAndEraseWorkCorrectly) { + base::test::TaskEnvironment task_envrionment; + + const std::vector<int> kValues = {1, 2, 3, 4, 5}; + const int kInsertLoopTimes = 10; + + VectorTester<int> vector_tester; + + // PushBack Values + for (auto value : kValues) { + vector_tester.PushBack(value); + vector_tester.Wait(); + auto insert_success = vector_tester.GetPushBackSuccess(); + ASSERT_TRUE(insert_success.has_value()); + EXPECT_TRUE(insert_success.value()); + } + + // Attempt to erase inserted values - should find one each. + for (auto value : kValues) { + vector_tester.Erase(value); + vector_tester.Wait(); + auto erase_success = vector_tester.GetEraseValue(); + ASSERT_TRUE(erase_success.has_value()); + EXPECT_EQ(erase_success.value(), uint64_t(1)); + } + + // Attempt to erase the values again - shouldn't find any. + for (auto value : kValues) { + vector_tester.Erase(value); + vector_tester.Wait(); + auto erase_success = vector_tester.GetEraseValue(); + ASSERT_TRUE(erase_success.has_value()); + EXPECT_EQ(erase_success.value(), uint64_t(0)); + } + + // Attempt to insert the values multiple times - should succeed. + for (int i = 0; i < kInsertLoopTimes; i++) { + for (auto value : kValues) { + vector_tester.PushBack(value); + vector_tester.Wait(); + auto insert_success = vector_tester.GetPushBackSuccess(); + ASSERT_TRUE(insert_success.has_value()); + EXPECT_TRUE(insert_success.value()); + } + } + + // Attempt to erase inserted values - should find kInsertLoopTimes each. + for (auto value : kValues) { + vector_tester.Erase(value); + vector_tester.Wait(); + auto erase_success = vector_tester.GetEraseValue(); + ASSERT_TRUE(erase_success.has_value()); + EXPECT_EQ(erase_success.value(), uint64_t(kInsertLoopTimes)); + } +} + +// Ensures that SharedVector::ExecuteIfFound works correctly +TEST(SharedVectorTest, ExecuteIfFoundSucceeds) { + base::test::TaskEnvironment task_envrionment; + + const int kExpectedValue = 1701; + const int kUnexpectedValue = 42; + + VectorTester<int> vector_tester; + vector_tester.PushBack(kExpectedValue); + vector_tester.Wait(); + + auto expected_finder = vector_tester.GetFinder(kExpectedValue); + vector_tester.ExecuteIfFound(&expected_finder); + expected_finder.Wait(); + auto found_result = expected_finder.found_result(); + ASSERT_TRUE(found_result.has_value()); + EXPECT_EQ(found_result.value(), kExpectedValue); + + auto unexpected_finder = vector_tester.GetFinder(kUnexpectedValue); + vector_tester.ExecuteIfFound(&unexpected_finder); + unexpected_finder.Wait(); + found_result = unexpected_finder.found_result(); + EXPECT_FALSE(found_result.has_value()); +} + +TEST(SharedVectorTest, ExecuteAllElements) { + base::test::TaskEnvironment task_envrionment; + + const std::vector<int> kValues = {1, 2, 3, 4, 5}; + + VectorTester<int> vector_tester; + + // PushBack Values + for (auto value : kValues) { + vector_tester.PushBack(value); + vector_tester.Wait(); + auto insert_success = vector_tester.GetPushBackSuccess(); + ASSERT_TRUE(insert_success.has_value()); + EXPECT_TRUE(insert_success.value()); + } + + auto executor = vector_tester.GetExecutor(kValues.size()); + vector_tester.ExecuteOnEachElement(&executor); + executor.Wait(); + EXPECT_EQ(executor.DifferenceInCount(), 0u); +} + +// Ensures that execution can happen on elements that are moveable but not +// copyable. +TEST(SharedVectorTest, InsertAndExecuteAndEraseOnUniquePtr) { + base::test::TaskEnvironment task_envrionment; + + const std::vector<int> kValues = {1, 2, 3, 4, 5}; + + VectorTester<std::unique_ptr<int>> vector_tester; + + for (auto value : kValues) { + vector_tester.PushBack(std::make_unique<int>(value)); + vector_tester.Wait(); + auto insert_success = vector_tester.GetPushBackSuccess(); + ASSERT_TRUE(insert_success.has_value()); + EXPECT_TRUE(insert_success.value()); + } + + auto executor = vector_tester.GetExecutor(kValues.size()); + vector_tester.ExecuteOnEachElement(&executor); + executor.Wait(); + EXPECT_EQ(executor.DifferenceInCount(), 0u); + + for (auto value : kValues) { + auto comparator_cb = base::BindRepeating( + [](int expected_value, const std::unique_ptr<int>& comparison_value) + -> bool { return expected_value == *comparison_value; }, + value); + vector_tester.Erase(comparator_cb); + vector_tester.Wait(); + auto erase_success = vector_tester.GetEraseValue(); + ASSERT_TRUE(erase_success.has_value()); + EXPECT_EQ(erase_success.value(), uint64_t(1)); + } +} + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/util/status.cc b/chromium/components/reporting/util/status.cc new file mode 100644 index 00000000000..418f1ef976f --- /dev/null +++ b/chromium/components/reporting/util/status.cc @@ -0,0 +1,123 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/status.h" + +#include <stdio.h> +#include <ostream> +#include <string> +#include <utility> + +#include "base/no_destructor.h" +#include "base/strings/strcat.h" +#include "components/reporting/util/status.pb.h" + +namespace reporting { +namespace error { +inline std::string CodeEnumToString(error::Code code) { + switch (code) { + case OK: + return "OK"; + case CANCELLED: + return "CANCELLED"; + case UNKNOWN: + return "UNKNOWN"; + case INVALID_ARGUMENT: + return "INVALID_ARGUMENT"; + case DEADLINE_EXCEEDED: + return "DEADLINE_EXCEEDED"; + case NOT_FOUND: + return "NOT_FOUND"; + case ALREADY_EXISTS: + return "ALREADY_EXISTS"; + case PERMISSION_DENIED: + return "PERMISSION_DENIED"; + case UNAUTHENTICATED: + return "UNAUTHENTICATED"; + case RESOURCE_EXHAUSTED: + return "RESOURCE_EXHAUSTED"; + case FAILED_PRECONDITION: + return "FAILED_PRECONDITION"; + case ABORTED: + return "ABORTED"; + case OUT_OF_RANGE: + return "OUT_OF_RANGE"; + case UNIMPLEMENTED: + return "UNIMPLEMENTED"; + case INTERNAL: + return "INTERNAL"; + case UNAVAILABLE: + return "UNAVAILABLE"; + case DATA_LOSS: + return "DATA_LOSS"; + } + + // No default clause, clang will abort if a code is missing from + // above switch. + return "UNKNOWN"; +} +} // namespace error. + +const Status& Status::StatusOK() { + static base::NoDestructor<Status> status_ok; + return *status_ok; +} + +Status::Status() : error_code_(error::OK) {} + +Status::Status(error::Code error_code, base::StringPiece error_message) + : error_code_(error_code) { + if (error_code != error::OK) { + error_message_ = std::string(error_message); + } +} + +Status::Status(const Status& other) + : error_code_(other.error_code_), error_message_(other.error_message_) {} + +Status& Status::operator=(const Status& other) { + error_code_ = other.error_code_; + error_message_ = other.error_message_; + return *this; +} + +bool Status::operator==(const Status& x) const { + return error_code_ == x.error_code_ && error_message_ == x.error_message_; +} + +std::string Status::ToString() const { + if (error_code_ == error::OK) { + return "OK"; + } + auto output = error::CodeEnumToString(error_code_); + if (!error_message_.empty()) { + base::StrAppend(&output, {":", error_message_}); + } + return output; +} + +void Status::SaveTo(StatusProto* status_proto) const { + status_proto->set_code(error_code_); + if (error_code_ != error::OK) { + status_proto->set_error_message(error_message_); + } else { + status_proto->clear_error_message(); + } +} + +void Status::RestoreFrom(const StatusProto& status_proto) { + error_code_ = static_cast<error::Code>(status_proto.code()); + if (error_code_ != error::OK) { + error_message_ = status_proto.error_message(); + } else { + error_message_.clear(); + } +} + +std::ostream& operator<<(std::ostream& os, const Status& x) { + os << x.ToString(); + return os; +} + +} // namespace reporting diff --git a/chromium/components/reporting/util/status.h b/chromium/components/reporting/util/status.h new file mode 100644 index 00000000000..c0a000d12d2 --- /dev/null +++ b/chromium/components/reporting/util/status.h @@ -0,0 +1,93 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_UTIL_STATUS_H_ +#define COMPONENTS_REPORTING_UTIL_STATUS_H_ + +#include <cstdint> +#include <iosfwd> +#include <string> + +#include "base/compiler_specific.h" +#include "base/strings/string_piece.h" +#include "components/reporting/util/status.pb.h" + +namespace reporting { +namespace error { +// These values must match error codes defined in google/rpc/code.proto +// (https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto) +enum Code : int32_t { + OK = 0, + CANCELLED = 1, + UNKNOWN = 2, + INVALID_ARGUMENT = 3, + DEADLINE_EXCEEDED = 4, + NOT_FOUND = 5, + ALREADY_EXISTS = 6, + PERMISSION_DENIED = 7, + UNAUTHENTICATED = 16, + RESOURCE_EXHAUSTED = 8, + FAILED_PRECONDITION = 9, + ABORTED = 10, + OUT_OF_RANGE = 11, + UNIMPLEMENTED = 12, + INTERNAL = 13, + UNAVAILABLE = 14, + DATA_LOSS = 15, +}; +} // namespace error + +class WARN_UNUSED_RESULT Status { + public: + // Creates a "successful" status. + Status(); + + // Create a status in the canonical error space with the specified + // code, and error message. If "code == 0", error_message is + // ignored and a Status object identical to Status::OK is + // constructed. + Status(error::Code error_code, base::StringPiece error_message); + Status(const Status&); + Status& operator=(const Status& x); + ~Status() = default; + + // Pre-defined Status object + static const Status& StatusOK(); + + // Accessor + bool ok() const { return error_code_ == error::OK; } + int error_code() const { return error_code_; } + error::Code code() const { return error_code_; } + base::StringPiece error_message() const { return error_message_; } + base::StringPiece message() const { return error_message_; } + + bool operator==(const Status& x) const; + bool operator!=(const Status& x) const { return !operator==(x); } + + // Return a combination of the error code name and message. + std::string ToString() const; + + // Exports the contents of this object into |status_proto|. This method sets + // all fields in |status_proto| (for OK status clears |error_message|). + void SaveTo(StatusProto* status_proto) const; + + // Populates this object using the contents of the given |status_proto|. + void RestoreFrom(const StatusProto& status_proto); + + private: + error::Code error_code_; + std::string error_message_; +}; + +// Prints a human-readable representation of 'x' to 'os'. +std::ostream& operator<<(std::ostream& os, const Status& x); + +#define CHECK_OK(value) CHECK((value).ok()) +#define DCHECK_OK(value) DCHECK((value).ok()) +#define ASSERT_OK(value) ASSERT_TRUE((value).ok()) +#define EXPECT_OK(value) EXPECT_TRUE((value).ok()) + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_UTIL_STATUS_H_ diff --git a/chromium/components/reporting/util/status.proto b/chromium/components/reporting/util/status.proto new file mode 100644 index 00000000000..6a307949388 --- /dev/null +++ b/chromium/components/reporting/util/status.proto @@ -0,0 +1,18 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package reporting; + +option optimize_for = LITE_RUNTIME; + +// Wire-format representation for a Status object. +message StatusProto { + // Numeric error code. + optional int32 code = 1; + + // Detailed error message explaining the error. + optional string error_message = 2; +}
\ No newline at end of file diff --git a/chromium/components/reporting/util/status_macros.h b/chromium/components/reporting/util/status_macros.h new file mode 100644 index 00000000000..6830b883c6d --- /dev/null +++ b/chromium/components/reporting/util/status_macros.h @@ -0,0 +1,81 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_UTIL_STATUS_MACROS_H_ +#define COMPONENTS_REPORTING_UTIL_STATUS_MACROS_H_ + +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" + +namespace reporting { + +// Run a command that returns a Status. If the called code returns an +// error status, return that status up out of this method too. +// +// Example: +// RETURN_IF_ERROR(DoThings(4)); +#define RETURN_IF_ERROR(expr) \ + do { \ + /* Using _status below to avoid capture problems if expr is "status". */ \ + const ::reporting::Status _status = (expr); \ + if (__builtin_expect(!_status.ok(), 0)) \ + return _status; \ + } while (0) + +// Internal helper for concatenating macro values. +#define STATUS_MACROS_CONCAT_NAME_INNER(x, y) x##y +#define STATUS_MACROS_CONCAT_NAME(x, y) STATUS_MACROS_CONCAT_NAME_INNER(x, y) + +#define ASSIGN_OR_RETURN_IMPL(result, lhs, rexpr) \ + auto result = rexpr; \ + if (__builtin_expect(!result.ok(), 0)) { \ + return result.status(); \ + } \ + lhs = std::move(result).ValueOrDie() + +// Executes an expression that returns a StatusOr, extracting its value +// into the variable defined by lhs (or returning on error). +// +// Example: Assigning to an existing value +// ValueType value; +// ASSIGN_OR_RETURN(value, MaybeGetValue(arg)); +// +// Example: Creating and assigning variable in one line. +// ASSIGN_OR_RETURN(ValueType value, MaybeGetValue(arg)); +// DoSomethingWithValueType(value); +// +// WARNING: ASSIGN_OR_RETURN expands into multiple statements; it cannot be used +// in a single statement (e.g. as the body of an if statement without {})! +#define ASSIGN_OR_RETURN(lhs, rexpr) \ + ASSIGN_OR_RETURN_IMPL( \ + STATUS_MACROS_CONCAT_NAME(_status_or_value, __COUNTER__), lhs, rexpr) + +#define ASSIGN_OR_ONCE_CALLBACK_AND_RETURN_IMPL(result, lhs, callback, rexpr) \ + const auto result = (rexpr); \ + if (__builtin_expect(!result.ok(), 0)) { \ + std::move(callback).Run(result.status()); \ + return; \ + } \ + lhs = result.ValueOrDie(); + +// Executes an expression that returns a StatusOr, extracting its value into the +// variabled defined by lhs (or calls callback with error and returns). +// +// Example: +// base::OnceCallback<void(Status)> callback = +// base::BindOnce([](Status status) {...}); +// ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(ValueType value, +// callback, +// MaybeGetValue(arg)); +// +// WARNING: ASSIGN_OR_RETURN expands into multiple statements; it cannot be used +// in a single statement (e.g. as the body of an if statement without {})! +#define ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(lhs, callback, rexpr) \ + ASSIGN_OR_ONCE_CALLBACK_AND_RETURN_IMPL( \ + STATUS_MACROS_CONCAT_NAME(_status_or_value, __COUNTER__), lhs, callback, \ + rexpr) + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_UTIL_STATUS_MACROS_H_ diff --git a/chromium/components/reporting/util/status_macros_unittest.cc b/chromium/components/reporting/util/status_macros_unittest.cc new file mode 100644 index 00000000000..7a6b3217df4 --- /dev/null +++ b/chromium/components/reporting/util/status_macros_unittest.cc @@ -0,0 +1,214 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/status_macros.h" + +#include <stdio.h> + +#include "base/bind.h" +#include "base/callback.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace reporting { +namespace { + +Status StatusTestFunction(bool fail) { + if (fail) { + return Status(error::INVALID_ARGUMENT, "Fail was true."); + } + return Status::StatusOK(); +} + +Status ReturnIfErrorWrapperFunction(bool fail) { + RETURN_IF_ERROR(StatusTestFunction(fail)); + + // Return error here to make sure that we aren't just returning the OK from + // StatusTestFunction. + return Status(error::INTERNAL, "Returning Internal Error"); +} + +// RETURN_IF_ERROR macro actually returns on a non-OK status. +TEST(StatusMacros, ReturnsOnError) { + Status test_status = ReturnIfErrorWrapperFunction(/*fail=*/true); + EXPECT_FALSE(test_status.ok()); + EXPECT_EQ(test_status.code(), error::INVALID_ARGUMENT); +} + +// RETURN_IF_ERROR macro continues on an OK status. +TEST(StatusMacros, ReturnIfErrorContinuesOnOk) { + Status test_status = ReturnIfErrorWrapperFunction(/*fail=*/false); + EXPECT_FALSE(test_status.ok()); + EXPECT_EQ(test_status.code(), error::INTERNAL); +} + +// Function to test StatusOr macros. +template <typename T> +StatusOr<T> StatusOrTestFunction(bool fail, T return_value) { + if (fail) { + return Status(error::INVALID_ARGUMENT, "Test failure requested."); + } + return std::forward<T>(return_value); +} + +// Function for testing ASSIGN_OR_RETURN. +template <typename T> +StatusOr<T> AssignOrReturnWrapperFunction(bool fail, T return_value) { + ASSIGN_OR_RETURN(T value, + StatusOrTestFunction(fail, std::forward<T>(return_value))); + return std::forward<T>(value); +} + +// ASSIGN_OR_RETURN actually assigns the value if the status is OK. +TEST(StatusMacros, AssignOnOk) { + constexpr int kReturnValue = 42; + + StatusOr<int> status_or_result = + AssignOrReturnWrapperFunction(/*fail=*/false, kReturnValue); + + ASSERT_TRUE(status_or_result.ok()); + EXPECT_EQ(status_or_result.ValueOrDie(), kReturnValue); +} + +// ASSIGN_OR_RETURN actually returns on a non-OK status. +TEST(StatusMacros, ReturnOnError) { + StatusOr<int> status_or_result = + AssignOrReturnWrapperFunction(/*fail=*/true, /*return_value=*/0); + EXPECT_FALSE(status_or_result.ok()); +} + +StatusOr<int> MultipleAssignOrReturnWrapperFunction(int return_value) { + bool fail = false; + int value; + ASSIGN_OR_RETURN(value, StatusOrTestFunction(fail, return_value)); + ASSIGN_OR_RETURN(value, StatusOrTestFunction(fail, return_value)); + ASSIGN_OR_RETURN(value, StatusOrTestFunction(fail, return_value)); + ASSIGN_OR_RETURN(value, StatusOrTestFunction(fail, return_value)); + return value; +} + +// ASSIGN_OR_RETURN compiles when used multiple times. +TEST(StatusMacros, MultipleAssignsSucceed) { + constexpr int kReturnValue = 42; + StatusOr<int> status_or_result = + MultipleAssignOrReturnWrapperFunction(kReturnValue); + ASSERT_TRUE(status_or_result.ok()); + EXPECT_EQ(status_or_result.ValueOrDie(), kReturnValue); +} + +// ASSIGN_OR_RETURN actually moves the value if the status is OK. +TEST(StatusMacros, AssignOnOkMoveable) { + constexpr int kReturnValue = 42; + + StatusOr<std::unique_ptr<int>> status_or_result = + AssignOrReturnWrapperFunction(/*fail=*/false, + std::make_unique<int>(kReturnValue)); + + ASSERT_TRUE(status_or_result.ok()); + EXPECT_EQ(*status_or_result.ValueOrDie(), kReturnValue); +} + +// ASSIGN_OR_RETURN actually returns on a non-OK status. +TEST(StatusMacros, ReturnOnErrorMoveable) { + StatusOr<std::unique_ptr<int>> status_or_result = + AssignOrReturnWrapperFunction(/*fail=*/true, + std::make_unique<int>(/*return_value=*/0)); + EXPECT_FALSE(status_or_result.ok()); +} + +// ASSIGN_OR_ONCE_CALLBACK_AND_RETURN testing +void AssignOrOnceCallbackWrapperFunction( + bool fail, + base::OnceCallback<void(Status)> callback) { + constexpr int kReturnValue = 42; + int value; + ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(value, callback, + StatusOrTestFunction(fail, kReturnValue)); + ASSERT_EQ(value, kReturnValue); +} + +class CallbackTestClass { + public: + explicit CallbackTestClass(Status test_status) : test_status_(test_status) {} + + void AssignInCallback(Status status) { + num_callback_invocations_++; + test_status_ = status; + } + + int num_callback_invocations() { return num_callback_invocations_; } + Status status() { return test_status_; } + + private: + Status test_status_; + int num_callback_invocations_ = 0; +}; + +// ASSIGN_OR_ONCE_CALLBACK_AND_RETURN assigns on OK error. +TEST(StatusMacros, OnceCallbackAssignOnOk) { + CallbackTestClass callback_test_class(Status::StatusOK()); + + base::OnceCallback<void(Status)> callback = + base::BindOnce(&CallbackTestClass::AssignInCallback, + base::Unretained(&callback_test_class)); + + AssignOrOnceCallbackWrapperFunction(/*fail=*/false, std::move(callback)); + + constexpr int kExpectedNumberOfCallbackInvocations = 0; + EXPECT_EQ(callback_test_class.num_callback_invocations(), + kExpectedNumberOfCallbackInvocations); + EXPECT_EQ(callback_test_class.status(), Status::StatusOK()); +} + +// ASSIGN_OR_ONCE_CALLBACK_AND_RETURN calls the callback and returns on non-OK +// status. +TEST(StatusMacros, OnceCallbackAndReturnOnError) { + CallbackTestClass callback_test_class(Status::StatusOK()); + + base::OnceCallback<void(Status)> callback = + base::BindOnce(&CallbackTestClass::AssignInCallback, + base::Unretained(&callback_test_class)); + + AssignOrOnceCallbackWrapperFunction(/*fail=*/true, std::move(callback)); + + constexpr int kExpectedNumberOfCallbackInvocations = 1; + EXPECT_EQ(callback_test_class.num_callback_invocations(), + kExpectedNumberOfCallbackInvocations); + EXPECT_EQ(callback_test_class.status().code(), error::INVALID_ARGUMENT); +} + +void MultipleAssignOrOnceCallbackWrapperFunction( + base::OnceCallback<void(Status)> callback) { + constexpr int kReturnValue = 42; + constexpr bool kFail = false; + + int value; + ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(value, callback, + StatusOrTestFunction(kFail, kReturnValue)); + ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(value, callback, + StatusOrTestFunction(kFail, kReturnValue)); + ASSIGN_OR_ONCE_CALLBACK_AND_RETURN(value, callback, + StatusOrTestFunction(kFail, kReturnValue)); + ASSERT_EQ(value, kReturnValue); +} + +// ASSIGN_OR_ONCE_CALLBACK_AND_RETURN can be used multiple times in a function. +TEST(StatusMacros, MultipleAssignOrOnceCallbackCompletes) { + CallbackTestClass callback_test_class(Status::StatusOK()); + + base::OnceCallback<void(Status)> callback = + base::BindOnce(&CallbackTestClass::AssignInCallback, + base::Unretained(&callback_test_class)); + + MultipleAssignOrOnceCallbackWrapperFunction(std::move(callback)); + + constexpr int kExpectedNumberOfCallbackInvocations = 0; + EXPECT_EQ(callback_test_class.num_callback_invocations(), + kExpectedNumberOfCallbackInvocations); + EXPECT_EQ(callback_test_class.status(), Status::StatusOK()); +} + +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/util/status_unittest.cc b/chromium/components/reporting/util/status_unittest.cc new file mode 100644 index 00000000000..2ba3b3d1a2c --- /dev/null +++ b/chromium/components/reporting/util/status_unittest.cc @@ -0,0 +1,192 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/status.h" + +#include <stdio.h> + +#include "base/logging.h" +#include "components/reporting/util/status.pb.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::StrEq; + +namespace reporting { +namespace { + +TEST(Status, Empty) { + Status status; + EXPECT_EQ(error::OK, status.error_code()); + EXPECT_EQ(error::OK, status.code()); + EXPECT_EQ("OK", status.ToString()); +} + +TEST(Status, GenericCodes) { + EXPECT_EQ(error::OK, Status::StatusOK().error_code()); + EXPECT_EQ(error::OK, Status::StatusOK().code()); + EXPECT_EQ("OK", Status::StatusOK().ToString()); +} + +TEST(Status, OkConstructorIgnoresMessage) { + Status status(error::OK, "msg"); + EXPECT_TRUE(status.ok()); + EXPECT_EQ("OK", status.ToString()); +} + +TEST(Status, CheckOK) { + Status status; + CHECK_OK(status); + CHECK_OK(status) << "Failed"; + DCHECK_OK(status) << "Failed"; +} + +TEST(Status, ErrorMessage) { + Status status(error::INVALID_ARGUMENT, ""); + EXPECT_FALSE(status.ok()); + EXPECT_EQ("", status.error_message()); + EXPECT_EQ("", status.message()); + EXPECT_EQ("INVALID_ARGUMENT", status.ToString()); + status = Status(error::INVALID_ARGUMENT, "msg"); + EXPECT_FALSE(status.ok()); + EXPECT_EQ("msg", status.error_message()); + EXPECT_EQ("msg", status.message()); + EXPECT_EQ("INVALID_ARGUMENT:msg", status.ToString()); + status = Status(error::OK, "msg"); + EXPECT_TRUE(status.ok()); + EXPECT_EQ("", status.error_message()); + EXPECT_EQ("", status.message()); + EXPECT_EQ("OK", status.ToString()); +} + +TEST(Status, Copy) { + Status a(error::UNKNOWN, "message"); + Status b(a); + EXPECT_EQ(a.ToString(), b.ToString()); +} + +TEST(Status, Assign) { + Status a(error::UNKNOWN, "message"); + Status b; + b = a; + EXPECT_EQ(a.ToString(), b.ToString()); +} + +TEST(Status, AssignEmpty) { + Status a(error::UNKNOWN, "message"); + Status b; + a = b; + EXPECT_EQ(std::string("OK"), a.ToString()); + EXPECT_TRUE(b.ok()); + EXPECT_TRUE(a.ok()); +} + +TEST(Status, EqualsOK) { + EXPECT_EQ(Status::StatusOK(), Status()); +} + +TEST(Status, EqualsSame) { + const Status a = Status(error::CANCELLED, "message"); + const Status b = Status(error::CANCELLED, "message"); + EXPECT_EQ(a, b); +} + +TEST(Status, EqualsCopy) { + const Status a = Status(error::CANCELLED, "message"); + const Status b = a; + EXPECT_EQ(a, b); +} + +TEST(Status, EqualsDifferentCode) { + const Status a = Status(error::CANCELLED, "message"); + const Status b = Status(error::UNKNOWN, "message"); + EXPECT_NE(a, b); +} + +TEST(Status, EqualsDifferentMessage) { + const Status a = Status(error::CANCELLED, "message"); + const Status b = Status(error::CANCELLED, "another"); + EXPECT_NE(a, b); +} + +TEST(Status, SaveOkTo) { + StatusProto status_proto; + Status::StatusOK().SaveTo(&status_proto); + + EXPECT_EQ(status_proto.code(), error::OK); + EXPECT_TRUE(status_proto.error_message().empty()) + << status_proto.error_message(); +} + +TEST(Status, SaveTo) { + Status status(error::INVALID_ARGUMENT, "argument error"); + StatusProto status_proto; + status.SaveTo(&status_proto); + + EXPECT_EQ(status_proto.code(), status.error_code()); + EXPECT_EQ(status_proto.error_message(), status.error_message()); +} + +TEST(Status, RestoreFromOk) { + StatusProto status_proto; + status_proto.set_code(error::OK); + status_proto.set_error_message("invalid error"); + + Status status; + status.RestoreFrom(status_proto); + + EXPECT_EQ(status.error_code(), status_proto.code()); + // Error messages are ignored for OK status objects. + EXPECT_TRUE(status.error_message().empty()) << status.error_message(); + EXPECT_TRUE(status.ok()); +} + +TEST(Status, RestoreFromNonOk) { + StatusProto status_proto; + status_proto.set_code(error::INVALID_ARGUMENT); + status_proto.set_error_message("argument error"); + + Status status; + status.RestoreFrom(status_proto); + + EXPECT_EQ(status.error_code(), status_proto.code()); + EXPECT_EQ(status.error_message(), status_proto.error_message()); +} + +TEST(Status, ConvertStatusToString) { + const std::pair<Status, const char*> status_pairs[] = { + {Status::StatusOK(), "OK"}, + {Status(error::CANCELLED, "Cancelled"), "CANCELLED:Cancelled"}, + {Status(error::UNKNOWN, "Unknown"), "UNKNOWN:Unknown"}, + {Status(error::INVALID_ARGUMENT, "Invalid argument"), + "INVALID_ARGUMENT:Invalid argument"}, + {Status(error::DEADLINE_EXCEEDED, "Deadline exceeded"), + "DEADLINE_EXCEEDED:Deadline exceeded"}, + {Status(error::NOT_FOUND, "Not found"), "NOT_FOUND:Not found"}, + {Status(error::ALREADY_EXISTS, "Already exists"), + "ALREADY_EXISTS:Already exists"}, + {Status(error::PERMISSION_DENIED, "Permission denied"), + "PERMISSION_DENIED:Permission denied"}, + {Status(error::UNAUTHENTICATED, "Unathenticated"), + "UNAUTHENTICATED:Unathenticated"}, + {Status(error::RESOURCE_EXHAUSTED, "Resourse exhausted"), + "RESOURCE_EXHAUSTED:Resourse exhausted"}, + {Status(error::FAILED_PRECONDITION, "Failed precondition"), + "FAILED_PRECONDITION:Failed precondition"}, + {Status(error::ABORTED, "Aborted"), "ABORTED:Aborted"}, + {Status(error::OUT_OF_RANGE, "Out of range"), + "OUT_OF_RANGE:Out of range"}, + {Status(error::UNIMPLEMENTED, "Unimplemented"), + "UNIMPLEMENTED:Unimplemented"}, + {Status(error::INTERNAL, "Internal"), "INTERNAL:Internal"}, + {Status(error::UNAVAILABLE, "Unavailable"), "UNAVAILABLE:Unavailable"}, + {Status(error::DATA_LOSS, "Data loss"), "DATA_LOSS:Data loss"}, + }; + for (const auto& p : status_pairs) { + LOG(INFO) << p.first; + EXPECT_THAT(p.first.ToString(), StrEq(p.second)); + } +} +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/util/statusor.cc b/chromium/components/reporting/util/statusor.cc new file mode 100644 index 00000000000..5bf2e3f92d2 --- /dev/null +++ b/chromium/components/reporting/util/statusor.cc @@ -0,0 +1,34 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/statusor.h" + +#include "base/logging.h" +#include "base/no_destructor.h" + +namespace reporting { +namespace internal { + +// static +const Status& StatusOrHelper::NotInitializedStatus() { + static base::NoDestructor<Status> status_not_initialized(error::UNKNOWN, + "Not initialized"); + return *status_not_initialized; +} + +// static +const Status& StatusOrHelper::MovedOutStatus() { + static base::NoDestructor<Status> status_moved_out(error::UNKNOWN, + "Value moved out"); + return *status_moved_out; +} + +// static +void StatusOrHelper::Crash(const Status& status) { + LOG(FATAL) << "Attempting to fetch value instead of handling error " + << status.ToString(); +} + +} // namespace internal +} // namespace reporting diff --git a/chromium/components/reporting/util/statusor.h b/chromium/components/reporting/util/statusor.h new file mode 100644 index 00000000000..5044da91d76 --- /dev/null +++ b/chromium/components/reporting/util/statusor.h @@ -0,0 +1,307 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// StatusOr<T> is the union of a Status object and a T +// object. StatusOr models the concept of an object that is either a +// usable value, or an error Status explaining why such a value is +// not present. To this end, StatusOr<T> does not allow its Status +// value to be Status::StatusOK(). Further, StatusOr<T*> does not allow the +// contained pointer to be nullptr. +// +// The primary use-case for StatusOr<T> is as the return value of a +// function which may fail. +// +// Example client usage for a StatusOr<T>, where T is not a pointer: +// +// StatusOr<float> result = DoBigCalculationThatCouldFail(); +// if (result.ok()) { +// float answer = result.ValueOrDie(); +// printf("Big calculation yielded: %f", answer); +// } else { +// LOG(ERROR) << result.status(); +// } +// +// Example client usage for a StatusOr<T*>: +// +// StatusOr<Foo*> result = FooFactory::MakeNewFoo(arg); +// if (result.ok()) { +// std::unique_ptr<Foo> foo(result.ValueOrDie()); +// foo->DoSomethingCool(); +// } else { +// LOG(ERROR) << result.status(); +// } +// +// Example client usage for a StatusOr<std::unique_ptr<T>>: +// +// StatusOr<std::unique_ptr<Foo>> result = FooFactory::MakeNewFoo(arg); +// if (result.ok()) { +// std::unique_ptr<Foo> foo = std::move(result.ValueOrDie()); +// foo->DoSomethingCool(); +// } else { +// LOG(ERROR) << result.status(); +// } +// +// Example factory implementation returning StatusOr<T*>: +// +// StatusOr<Foo*> FooFactory::MakeNewFoo(int arg) { +// if (arg <= 0) { +// return Status(error::INVALID_ARGUMENT, "Arg must be positive"); +// } else { +// return new Foo(arg); +// } +// } +// + +#ifndef COMPONENTS_REPORTING_UTIL_STATUSOR_H_ +#define COMPONENTS_REPORTING_UTIL_STATUSOR_H_ + +#include <new> +#include <string> +#include <type_traits> +#include <utility> + +#include "base/compiler_specific.h" +#include "base/logging.h" +#include "base/optional.h" +#include "components/reporting/util/status.h" + +namespace reporting { + +namespace internal { + +// Helper class for StatusOr to use. +class StatusOrHelper { + public: + static const Status& NotInitializedStatus(); + static const Status& MovedOutStatus(); + static void Crash(const Status& status); +}; + +} // namespace internal + +template <typename T> +class WARN_UNUSED_RESULT StatusOr { + template <typename U> + friend class StatusOr; + + // A traits class that determines whether a type U is implicitly convertible + // from a type V. If it is convertible, then the |value| member of this class + // is statically set to true, otherwise it is statically set to false. + template <class U, typename V> + struct is_implicitly_constructible + : base::conjunction<std::is_constructible<U, V>, + std::is_convertible<V, U>> {}; + + public: + // Constructs a new StatusOr with UNINITIALIZED status and no value. + StatusOr() : status_(internal::StatusOrHelper::NotInitializedStatus()) {} + + // Constructs a new StatusOr with the given non-ok status. After calling + // this constructor, calls to ValueOrDie() will CHECK-fail. + // + // This constructor is not declared explicit so that a function with a return + // type of |StatusOr<T>| can return a Status object, and the status will be + // implicitly converted to the appropriate return type as a matter of + // convenience. + // REQUIRES: !status.ok(). + StatusOr(const Status& status) // NOLINT(runtime/explicit) + : status_(status) { + if (status.ok()) { + internal::StatusOrHelper::Crash(status); + } + } + + // Constructs a StatusOr object that contains |value|. The resulting object + // is considered to have an OK status. The wrapped element can be accessed + // with ValueOrDie(). + // + // This constructor is made implicit so that a function with a return type of + // |StatusOr<T>| can return an object of type |U&&|, implicitly converting + // it to a |StatusOr<T>| object. + // + // Note that |T| must be implicitly constructible from |U|, and |U| must not + // be a (cv-qualified) Status or Status-reference type. Due to C++ + // reference-collapsing rules and perfect-forwarding semantics, this + // constructor matches invocations that pass |value| either as a const + // reference or as an rvalue reference. Since StatusOr needs to work for both + // reference and rvalue-reference types, the constructor uses perfect + // forwarding to avoid invalidating arguments that were passed by reference. + template <typename U, + typename E = typename std::enable_if< + is_implicitly_constructible<T, U>::value && + !std::is_same<typename std::remove_reference< + typename std::remove_cv<U>::type>::type, + Status>::value>::type> + StatusOr(U&& value) // NOLINT(runtime/explicit) + : status_(Status::StatusOK()), value_(std::forward<U>(value)) {} + + // Copy constructor. + // + // This constructor needs to be explicitly defined because the presence of + // the move-assignment operator deletes the default copy constructor. In such + // a scenario, since the deleted copy constructor has stricter binding rules + // than the templated copy constructor, the templated constructor cannot act + // as a copy constructor, and any attempt to copy-construct a |StatusOr| + // object results in a compilation error. + StatusOr(const StatusOr& other) + : status_(other.status_), value_(other.value_) {} + + // Templatized constructor that constructs a |StatusOr<T>| from a const + // reference to a |StatusOr<U>|. + // + // |T| must be implicitly constructible from |const U&|. + template <typename U, + typename E = typename std::enable_if< + is_implicitly_constructible<T, const U&>::value>::type> + StatusOr(const StatusOr<U>& other) // NOLINT(runtime/explicit) + : status_(other.status_), value_(other.value_) {} + + // Copy-assignment operator. + StatusOr& operator=(const StatusOr& other) { + // Check for self-assignment. + if (this == &other) { + return *this; + } + + if (other.status_.ok()) { + AssignValue(other.value_); + } else { + AssignStatus(other.status_); + } + return *this; + } + + // Templatized constructor which constructs a |StatusOr<T>| by moving the + // contents of a |StatusOr<U>|. |T| must be implicitly constructible from + // |U&&|. + // + // Sets |other| to contain a non-OK status with a|error::UNKNOWN| + // error code. + template <typename U, + typename E = typename std::enable_if< + is_implicitly_constructible<T, U&&>::value>::type> + StatusOr(StatusOr<U>&& other) // NOLINT(runtime/explicit) + : status_(std::move(other.status_)), value_(std::move(other.value_)) { + other.status_ = internal::StatusOrHelper::MovedOutStatus(); + } + + // Move-assignment operator. + // + // Sets |other| to contain a non-OK status with a |error::UNKNOWN| error + // code. + StatusOr& operator=(StatusOr&& other) { + // Check for self-assignment. + if (this == &other) { + return *this; + } + + if (other.status_.ok()) { + AssignValue(std::move(other.value_.value())); + } else { + AssignStatus(std::move(other.status_)); + } + other.status_ = internal::StatusOrHelper::MovedOutStatus(); + + return *this; + } + + // Indicates whether the object contains a |T| value. + bool ok() const { return status_.ok(); } + + // Gets the stored status object, or an OK status if a |T| value is stored. + Status status() const { return status_; } + + // Gets the stored |T| value. + // + // This method should only be called if this StatusOr object's status is OK + // (i.e. a call to ok() returns true), otherwise this call will abort. + const T& WARN_UNUSED_RESULT ValueOrDie() const& { + if (!ok()) { + internal::StatusOrHelper::Crash(status_); + } + return value_.value(); + } + + // Gets a mutable reference to the stored |T| value. + // + // This method should only be called if this StatusOr object's status is OK + // (i.e. a call to ok() returns true), otherwise this call will abort. + T& WARN_UNUSED_RESULT ValueOrDie() & { + if (!ok()) { + internal::StatusOrHelper::Crash(status_); + } + return value_.value(); + } + + // Moves and returns the internally-stored |T| value. + // + // This method should only be called if this StatusOr object's status is OK + // (i.e. a call to ok() returns true), otherwise this call will abort. The + // StatusOr object is invalidated after this call and will be updated to + // contain a non-OK status with a |error::UNKNOWN| error code. + T WARN_UNUSED_RESULT ValueOrDie() && { + if (!ok()) { + internal::StatusOrHelper::Crash(status_); + } + + // Invalidate this StatusOr object before returning control to caller. + StatusResetter set_moved_status(this, + internal::StatusOrHelper::MovedOutStatus()); + return std::move(value_.value()); + } + + private: + class StatusResetter { + public: + StatusResetter(StatusOr<T>* status_or, const Status& reset_to_status) + : status_or_(status_or), reset_to_status_(reset_to_status) {} + StatusResetter(const StatusResetter& other) = delete; + StatusResetter& operator=(const StatusResetter& other) = delete; + ~StatusResetter() { + status_or_->OverwriteValueWithStatus(reset_to_status_); + } + + private: + StatusOr<T>* const status_or_; + const Status reset_to_status_; + }; + + // Resets |this| to contain |status|. + template <class U> + void AssignStatus(U&& status) { + if (ok()) { + OverwriteValueWithStatus(std::forward<U>(status)); + } else { + status_ = std::forward<U>(status); + } + } + + // Under the assumption that |this| is currently holding a value, resets the + // |value_| member and sets |status_| to indicate that |this| does not have + // a value. + template <class U> + void OverwriteValueWithStatus(U&& status) { + if (!ok()) { + LOG(FATAL) << "Object does not have a value to change from"; + } + value_.reset(); + status_ = std::forward<U>(status); + } + + // Resets |value_| to contain the |value| and sets |status_| + // to OK, indicating that the StatusOr object has a value. + // Destroys the existing |value_|. + template <class U> + void AssignValue(U&& value) { + value_ = std::forward<U>(value); + status_ = Status::StatusOK(); + } + + Status status_; + base::Optional<T> value_; +}; + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_UTIL_STATUSOR_H_ diff --git a/chromium/components/reporting/util/statusor_unittest.cc b/chromium/components/reporting/util/statusor_unittest.cc new file mode 100644 index 00000000000..38736feb76d --- /dev/null +++ b/chromium/components/reporting/util/statusor_unittest.cc @@ -0,0 +1,286 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/statusor.h" + +#include <errno.h> +#include <algorithm> +#include <memory> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" + +#include "testing/gtest/include/gtest/gtest-death-test.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace reporting { +namespace { + +class Base1 { + public: + virtual ~Base1() = default; + int pad; +}; + +class Base2 { + public: + virtual ~Base2() = default; + int yetotherpad; +}; + +class Derived : public Base1, public Base2 { + public: + ~Derived() override = default; + int evenmorepad; +}; + +class CopyNoAssign { + public: + explicit CopyNoAssign(int value) : foo(value) {} + CopyNoAssign(const CopyNoAssign& other) : foo(other.foo) {} + int foo; + + private: + const CopyNoAssign& operator=(const CopyNoAssign&); +}; + +TEST(StatusOr, TestDefaultCtor) { + StatusOr<int> thing; + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(error::UNKNOWN, thing.status().code()); +} + +TEST(StatusOr, TestStatusCtor) { + StatusOr<int> thing(Status(error::CANCELLED, "")); + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(Status(error::CANCELLED, ""), thing.status()); +} + +TEST(StatusOr, TestValueCtor) { + const int kI = 4; + StatusOr<int> thing(kI); + EXPECT_TRUE(thing.ok()); + EXPECT_EQ(kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestCopyCtorStatusOk) { + const int kI = 4; + StatusOr<int> original(kI); + StatusOr<int> copy(original); + EXPECT_EQ(original.status(), copy.status()); + EXPECT_EQ(original.ValueOrDie(), copy.ValueOrDie()); +} + +TEST(StatusOr, TestCopyCtorStatusNotOk) { + StatusOr<int> original(Status(error::CANCELLED, "")); + StatusOr<int> copy(original); + EXPECT_EQ(original.status(), copy.status()); +} + +TEST(StatusOr, TestCopyCtorStatusOKConverting) { + const int kI = 4; + StatusOr<int> original(kI); + StatusOr<double> copy(original); + EXPECT_EQ(original.status(), copy.status()); + EXPECT_EQ(original.ValueOrDie(), copy.ValueOrDie()); +} + +TEST(StatusOr, TestCopyCtorStatusNotOkConverting) { + StatusOr<int> original(Status(error::CANCELLED, "")); + StatusOr<double> copy(original); + EXPECT_EQ(original.status(), copy.status()); +} + +TEST(StatusOr, TestAssignmentStatusOk) { + const int kI = 4; + StatusOr<int> source(kI); + StatusOr<int> target; + target = source; + EXPECT_EQ(source.status(), target.status()); + EXPECT_EQ(source.ValueOrDie(), target.ValueOrDie()); +} + +TEST(StatusOr, TestAssignmentStatusNotOk) { + StatusOr<int> source(Status(error::CANCELLED, "")); + StatusOr<int> target; + target = source; + EXPECT_EQ(source.status(), target.status()); +} + +TEST(StatusOr, TestAssignmentStatusOKConverting) { + const int kI = 4; + StatusOr<int> source(kI); + StatusOr<double> target; + target = source; + EXPECT_EQ(source.status(), target.status()); + EXPECT_DOUBLE_EQ(source.ValueOrDie(), target.ValueOrDie()); +} + +TEST(StatusOr, TestAssignmentStatusNotOkConverting) { + StatusOr<int> source(Status(error::CANCELLED, "")); + StatusOr<double> target; + target = source; + EXPECT_EQ(source.status(), target.status()); +} + +TEST(StatusOr, TestStatus) { + StatusOr<int> good(4); + EXPECT_TRUE(good.ok()); + StatusOr<int> bad(Status(error::CANCELLED, "")); + EXPECT_FALSE(bad.ok()); + EXPECT_EQ(Status(error::CANCELLED, ""), bad.status()); +} + +TEST(StatusOr, TestValueConst) { + const int kI = 4; + const StatusOr<int> thing(kI); + EXPECT_EQ(kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestPointerDefaultCtor) { + StatusOr<int*> thing; + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(error::UNKNOWN, thing.status().code()); +} + +TEST(StatusOr, TestPointerStatusCtor) { + StatusOr<int*> thing(Status(error::CANCELLED, "")); + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(Status(error::CANCELLED, ""), thing.status()); +} + +TEST(StatusOr, TestPointerValueCtor) { + const int kI = 4; + StatusOr<const int*> thing(&kI); + EXPECT_TRUE(thing.ok()); + EXPECT_EQ(&kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusOk) { + const int kI = 0; + StatusOr<const int*> original(&kI); + StatusOr<const int*> copy(original); + EXPECT_EQ(original.status(), copy.status()); + EXPECT_EQ(original.ValueOrDie(), copy.ValueOrDie()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusNotOk) { + StatusOr<int*> original(Status(error::CANCELLED, "")); + StatusOr<int*> copy(original); + EXPECT_EQ(original.status(), copy.status()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusOKConverting) { + Derived derived; + StatusOr<Derived*> original(&derived); + StatusOr<Base2*> copy(original); + EXPECT_EQ(original.status(), copy.status()); + EXPECT_EQ(static_cast<const Base2*>(original.ValueOrDie()), + copy.ValueOrDie()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusNotOkConverting) { + StatusOr<Derived*> original(Status(error::CANCELLED, "")); + StatusOr<Base2*> copy(original); + EXPECT_EQ(original.status(), copy.status()); +} + +TEST(StatusOr, TestPointerAssignmentStatusOk) { + const int kI = 0; + StatusOr<const int*> source(&kI); + StatusOr<const int*> target; + target = source; + EXPECT_EQ(source.status(), target.status()); + EXPECT_EQ(source.ValueOrDie(), target.ValueOrDie()); +} + +TEST(StatusOr, TestPointerAssignmentStatusNotOk) { + StatusOr<int*> source(Status(error::CANCELLED, "")); + StatusOr<int*> target; + target = source; + EXPECT_EQ(source.status(), target.status()); +} + +TEST(StatusOr, TestPointerAssignmentStatusOKConverting) { + Derived derived; + StatusOr<Derived*> source(&derived); + StatusOr<Base2*> target; + target = source; + EXPECT_EQ(source.status(), target.status()); + EXPECT_EQ(static_cast<const Base2*>(source.ValueOrDie()), + target.ValueOrDie()); +} + +TEST(StatusOr, TestPointerAssignmentStatusNotOkConverting) { + StatusOr<Derived*> source(Status(error::CANCELLED, "")); + StatusOr<Base2*> target; + target = source; + EXPECT_EQ(source.status(), target.status()); +} + +TEST(StatusOr, TestPointerStatus) { + const int kI = 0; + StatusOr<const int*> good(&kI); + EXPECT_TRUE(good.ok()); + StatusOr<const int*> bad(Status(error::CANCELLED, "")); + EXPECT_EQ(Status(error::CANCELLED, ""), bad.status()); +} + +TEST(StatusOr, TestPointerValue) { + const int kI = 0; + StatusOr<const int*> thing(&kI); + EXPECT_EQ(&kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestPointerValueConst) { + const int kI = 0; + const StatusOr<const int*> thing(&kI); + EXPECT_EQ(&kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestMoveStatusOr) { + const int kI = 0; + StatusOr<std::unique_ptr<int>> thing(std::make_unique<int>(kI)); + EXPECT_OK(thing.status()); + StatusOr<std::unique_ptr<int>> moved = std::move(thing); + EXPECT_EQ(error::UNKNOWN, thing.status().code()); + EXPECT_TRUE(moved.ok()); + EXPECT_EQ(kI, *moved.ValueOrDie()); +} + +TEST(StatusOr, TestBinding) { + class RefCountedValue : public base::RefCounted<RefCountedValue> { + public: + explicit RefCountedValue(StatusOr<int> value) : value_(value) {} + Status status() const { return value_.status(); } + int value() const { return value_.ValueOrDie(); } + + private: + friend class base::RefCounted<RefCountedValue>; + ~RefCountedValue() = default; + const StatusOr<int> value_; + }; + const int kI = 0; + base::OnceCallback<int(StatusOr<scoped_refptr<RefCountedValue>>)> callback = + base::BindOnce([](StatusOr<scoped_refptr<RefCountedValue>> val) { + return val.ValueOrDie()->value(); + }); + const int result = + std::move(callback).Run(base::MakeRefCounted<RefCountedValue>(kI)); + EXPECT_EQ(kI, result); +} + +TEST(StatusOr, TestAbort) { + StatusOr<int> thing1(Status(error::UNKNOWN, "Unknown")); + int v1; + EXPECT_DEATH_IF_SUPPORTED(v1 = thing1.ValueOrDie(), ""); + + StatusOr<std::unique_ptr<int>> thing2(Status(error::UNKNOWN, "Unknown")); + std::unique_ptr<int> v2; + EXPECT_DEATH_IF_SUPPORTED(v2 = std::move(thing2.ValueOrDie()), ""); +} +} // namespace +} // namespace reporting diff --git a/chromium/components/reporting/util/task_runner_context.h b/chromium/components/reporting/util/task_runner_context.h new file mode 100644 index 00000000000..8c97ad1fb90 --- /dev/null +++ b/chromium/components/reporting/util/task_runner_context.h @@ -0,0 +1,175 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_REPORTING_UTIL_TASK_RUNNER_CONTEXT_H_ +#define COMPONENTS_REPORTING_UTIL_TASK_RUNNER_CONTEXT_H_ + +#include <utility> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/callback_helpers.h" +#include "base/logging.h" +#include "base/memory/scoped_refptr.h" +#include "base/sequence_checker.h" +#include "base/sequenced_task_runner.h" +#include "base/time/time.h" + +namespace reporting { + +// This class defines context for multiple actions executed on a sequenced task +// runner with the ability to make asynchronous calls to other threads and +// resuming sequenced execution by calling |Schedule| or |ScheduleAfter|. +// Multiple actions can be scheduled at once; they will be executed on the same +// sequenced task runner. Ends execution and self-destructs when one of the +// actions calls |Response| (all previously scheduled actions must be completed +// or cancelled by then, otherwise they will crash). +// +// Code snippet: +// +// Declaration: +// class SeriesOfActionsContext { +// public: +// SeriesOfActionsContext( +// ..., +// base::OnceCallback<void(...)> callback, +// scoped_refptr<base::SequencedTaskRunner> task_runner) +// : TaskRunnerContext<...>(std::move(callback), +// std::move(task_runner)) {} +// +// private: +// // Context can only be deleted by calling Response method. +// ~SeriesOfActionsContext() override = default; +// +// void Action1(...) { +// ... +// if (...) { +// Response(...); +// return; +// } +// Schedule(&SeriesOfActionsContext::Action2, +// base::Unretained(this), +// ...); +// ... +// ScheduleAfter(delay, +// &SeriesOfActionsContext::Action3, +// base::Unretained(this), +// ...); +// } +// +// void OnStart() override { Action1(...); } +// }; +// +// Usage: +// Start<SeriesOfActionsContext>( +// ..., +// returning_callback, +// base::SequencedTaskRunnerHandle::Get()); +// +template <typename ResponseType> +class TaskRunnerContext { + public: + TaskRunnerContext(const TaskRunnerContext& other) = delete; + TaskRunnerContext& operator=(const TaskRunnerContext& other) = delete; + + // Schedules next execution (can be called from any thread). + template <class Function, class... Args> + void Schedule(Function&& proc, Args&&... args) { + task_runner_->PostTask(FROM_HERE, + base::BindOnce(std::forward<Function>(proc), + std::forward<Args>(args)...)); + } + + // Schedules next execution with delay (can be called from any thread). + template <class Function, class... Args> + void ScheduleAfter(base::TimeDelta delay, Function&& proc, Args&&... args) { + task_runner_->PostDelayedTask(FROM_HERE, + base::BindOnce(std::forward<Function>(proc), + std::forward<Args>(args)...), + delay); + } + + // Responds to the caller once completed the work sequence + // (can only be called by action scheduled to the sequenced task runner). + void Response(ResponseType result) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + OnCompletion(); + + // Respond to the caller. + DCHECK(!callback_.is_null()) << "Already responded"; + std::move(callback_).Run(std::forward<ResponseType>(result)); + + // Self-destruct. + delete this; + } + + // Helper method checks that the caller runs on valid sequence. + // Can be used by any scheduled action. + // No need to call it by OnStart, OnCompletion and destructor. + // For non-debug builds it is a no-op. + void CheckOnValidSequence() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + } + + protected: + // Constructor is protected, for derived class to refer to. + TaskRunnerContext(base::OnceCallback<void(ResponseType)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : callback_(std::move(callback)), task_runner_(std::move(task_runner)) { + // Constructor can be called from any thread. + DETACH_FROM_SEQUENCE(sequence_checker_); + } + + // Context can only be deleted by calling Response method. + virtual ~TaskRunnerContext() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(callback_.is_null()) << "Deleted without responding to the caller"; + } + + private: + template <typename ContextType /* derived from TaskRunnerContext*/, + class... Args> + friend void Start(Args&&... args); + + // Hook for execution start. Should be overridden to do non-trivial work. + virtual void OnStart() { Response(ResponseType()); } + + // Finalization action before responding and deleting the context. + // May be overridden, if necessary. + virtual void OnCompletion() {} + + // Wrapper for OnStart to mandate sequence checker. + void OnStartWrap() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + OnStart(); + } + + // User callback to deliver result. + base::OnceCallback<void(ResponseType)> callback_; + + // Sequential task runner (guarantees that each action is executed + // sequentially in order of submission). + scoped_refptr<base::SequencedTaskRunner> task_runner_; + + SEQUENCE_CHECKER(sequence_checker_); +}; + +// Constructs the context and starts execution on the assigned sequential task +// runner. Can be called from any thread to schedule the first action in the +// sequence. +template <typename ContextType /* derived from TaskRunnerContext*/, + class... Args> +void Start(Args&&... args) { + ContextType* const context = new ContextType(std::forward<Args>(args)...); + // Start execution handing |context| over to the callback, in order + // to make sure final |OnStart| (with possible |Response| and self-destruct) + // can only happen on |task_runner|. + context->task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&ContextType::OnStartWrap, base::Unretained(context))); +} + +} // namespace reporting + +#endif // COMPONENTS_REPORTING_UTIL_TASK_RUNNER_CONTEXT_H_ diff --git a/chromium/components/reporting/util/task_runner_context_unittest.cc b/chromium/components/reporting/util/task_runner_context_unittest.cc new file mode 100644 index 00000000000..ff93922904b --- /dev/null +++ b/chromium/components/reporting/util/task_runner_context_unittest.cc @@ -0,0 +1,433 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/reporting/util/task_runner_context.h" + +#include <functional> +#include <memory> +#include <vector> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "base/sequenced_task_runner.h" +#include "base/synchronization/waitable_event.h" +#include "base/test/task_environment.h" +#include "base/threading/sequenced_task_runner_handle.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace reporting { +namespace { + +class TaskRunner : public ::testing::Test { + protected: + base::test::TaskEnvironment task_environment_; +}; + +// This is the simplest test - runs one action only on a sequenced task runner. +TEST_F(TaskRunner, SingleAction) { + class SingleActionContext : public TaskRunnerContext<bool> { + public: + SingleActionContext(base::OnceCallback<void(bool)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : TaskRunnerContext<bool>(std::move(callback), std::move(task_runner)) { + } + + private: + void OnStart() override { Response(true); } + }; + + bool result = false; + // Created context reference is self-destruct upon completion of 'Start', + // but the context itself lives through until all tasks are done. + base::RunLoop run_loop; + Start<SingleActionContext>( + base::BindOnce( + [](base::RunLoop* run_loop, bool* var, bool val) { + *var = val; + run_loop->Quit(); + }, + &run_loop, &result), + base::SequencedTaskRunnerHandle::Get()); + run_loop.Run(); + EXPECT_TRUE(result); +} + +// This test runs a series of action on a sequenced task runner. +TEST_F(TaskRunner, SeriesOfActions) { + class SeriesOfActionsContext : public TaskRunnerContext<uint32_t> { + public: + SeriesOfActionsContext(uint32_t init_value, + base::OnceCallback<void(uint32_t)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : TaskRunnerContext<uint32_t>(std::move(callback), + std::move(task_runner)), + init_value_(init_value) {} + + private: + void Halve(uint32_t value, uint32_t log) { + CheckOnValidSequence(); + if (value <= 1) { + Response(log); + return; + } + Schedule(&SeriesOfActionsContext::Halve, base::Unretained(this), + value / 2, log + 1); + } + + void OnStart() override { Halve(init_value_, 0); } + + const uint32_t init_value_; + }; + + uint32_t result = 0; + base::RunLoop run_loop; + Start<SeriesOfActionsContext>( + 128, + base::BindOnce( + [](base::RunLoop* run_loop, uint32_t* var, uint32_t val) { + *var = val; + run_loop->Quit(); + }, + &run_loop, &result), + base::SequencedTaskRunnerHandle::Get()); + run_loop.Run(); + EXPECT_EQ(result, 7u); +} + +// This test runs the same series of actions injecting delays. +TEST_F(TaskRunner, SeriesOfDelays) { + class SeriesOfDelaysContext : public TaskRunnerContext<uint32_t> { + public: + SeriesOfDelaysContext(uint32_t init_value, + base::OnceCallback<void(uint32_t)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : TaskRunnerContext<uint32_t>(std::move(callback), + std::move(task_runner)), + init_value_(init_value), + delay_(base::TimeDelta::FromSecondsD(0.1)) {} + + private: + void Halve(uint32_t value, uint32_t log) { + CheckOnValidSequence(); + if (value <= 1) { + Response(log); + return; + } + delay_ += base::TimeDelta::FromSecondsD(0.1); + ScheduleAfter(delay_, &SeriesOfDelaysContext::Halve, + base::Unretained(this), value / 2, log + 1); + } + + void OnStart() override { Halve(init_value_, 0); } + + const uint32_t init_value_; + base::TimeDelta delay_; + }; + + // Run on another thread, so that we can wait on the quit event here + // and avoid RunLoopIdle (which would exit on the first delay). + uint32_t result = 0; + base::RunLoop run_loop; + Start<SeriesOfDelaysContext>( + 128, + base::BindOnce( + [](base::RunLoop* run_loop, uint32_t* var, uint32_t val) { + *var = val; + run_loop->Quit(); + }, + &run_loop, &result), + base::SequencedTaskRunnerHandle::Get()); + run_loop.Run(); + EXPECT_EQ(result, 7u); +} + +// This test runs the same series of actions offsetting them to a random threads +// and then taking control back to the sequenced task runner. +TEST_F(TaskRunner, SeriesOfAsyncs) { + class SeriesOfAsyncsContext : public TaskRunnerContext<uint32_t> { + public: + SeriesOfAsyncsContext(uint32_t init_value, + base::OnceCallback<void(uint32_t)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : TaskRunnerContext<uint32_t>(std::move(callback), + std::move(task_runner)), + init_value_(init_value), + delay_(base::TimeDelta::FromSecondsD(0.1)) {} + + private: + void Halve(uint32_t value, uint32_t log) { + CheckOnValidSequence(); + if (value <= 1) { + Response(log); + return; + } + // Perform a calculation on a generic thread pool with delay, + // then get back to the sequence by calling Schedule from there. + delay_ += base::TimeDelta::FromSecondsD(0.1); + base::ThreadPool::PostDelayedTask( + FROM_HERE, + base::BindOnce( + [](uint32_t value, uint32_t log, SeriesOfAsyncsContext* context) { + // Action executed asyncrhonously. + value /= 2; + ++log; + // Getting back to the sequence. + context->Schedule(&SeriesOfAsyncsContext::Halve, + base::Unretained(context), value, log); + }, + value, log, base::Unretained(this)), + delay_); + } + + void OnStart() override { Halve(init_value_, 0); } + + const uint32_t init_value_; + base::TimeDelta delay_; + }; + + // Run on another thread, so that we can wait on the quit event here + // and avoid RunLoopIdle (which would exit on the first delay). + uint32_t result = 0; + base::RunLoop run_loop; + Start<SeriesOfAsyncsContext>( + 128, + base::BindOnce( + [](base::RunLoop* run_loop, uint32_t* var, uint32_t val) { + *var = val; + run_loop->Quit(); + }, + &run_loop, &result), + base::SequencedTaskRunnerHandle::Get()); + + run_loop.Run(); + EXPECT_EQ(result, 7u); +} + +// This test calculates Fibonacci as a tree of recurrent actions on a sequenced +// task runner. Note that 2 actions are scheduled in parallel. +TEST_F(TaskRunner, TreeOfActions) { + // Helper class accepts multiple 'AddIncoming' calls to add numbers, + // and invokes 'callback' when last reference to it is dropped. + class Summator : public base::RefCounted<Summator> { + public: + explicit Summator(base::OnceCallback<void(uint32_t)> callback) + : callback_(std::move(callback)) {} + + void AddIncoming(uint32_t incoming) { result_ += incoming; } + + protected: + virtual ~Summator() { + DCHECK(!callback_.is_null()); + std::move(callback_).Run(result_); + } + + private: + friend class base::RefCounted<Summator>; + + uint32_t result_ = 0; + base::OnceCallback<void(uint32_t)> callback_; + }; + + // Context class for Fibonacci asynchronous recursion tree. + class TreeOfActionsContext : public TaskRunnerContext<uint32_t> { + public: + TreeOfActionsContext(uint32_t init_value, + base::OnceCallback<void(uint32_t)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : TaskRunnerContext<uint32_t>(std::move(callback), + std::move(task_runner)), + init_value_(init_value) {} + + private: + void FibonacciSplit(uint32_t value, scoped_refptr<Summator> join) { + CheckOnValidSequence(); + if (value < 2u) { + join->AddIncoming(value); // Fib(0) == 1, Fib(1) == 1 + return; // No more actions to schedule. + } + // Schedule two asynchronous recursive calls. + // 'join' above will self-destruct once both callbacks complete + // and drop references to it. Each callback spawns additional + // callbacks, and when they complete, adds the results to its + // own 'Summator' instance. + for (const uint32_t subval : {value - 1, value - 2}) { + Schedule(&TreeOfActionsContext::FibonacciSplit, base::Unretained(this), + subval, + base::MakeRefCounted<Summator>( + base::BindOnce(&Summator::AddIncoming, join))); + } + } + + void OnStart() override { + FibonacciSplit(init_value_, base::MakeRefCounted<Summator>(base::BindOnce( + &TreeOfActionsContext::Response, + base::Unretained(this)))); + } + + const uint32_t init_value_; + }; + + const std::vector<uint32_t> expected_fibo_results( + {0, 1, 1, 2, 3, 5, 8, 13, 21, 34, + 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181}); + std::vector<uint32_t> actual_fibo_results(expected_fibo_results.size()); + base::RunLoop run_loop; + size_t count = expected_fibo_results.size(); + // Start all calculations (they will intermix on the same sequential runner). + for (uint32_t n = 0; n < expected_fibo_results.size(); ++n) { + uint32_t* const result = &actual_fibo_results[n]; + *result = 0; + base::SequencedTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce( + [](size_t* count, base::RunLoop* run_loop, uint32_t n, + uint32_t* result) { + Start<TreeOfActionsContext>( + n, + base::BindOnce( + [](size_t* count, base::RunLoop* run_loop, + uint32_t* var, uint32_t val) { + *var = val; + if (!--*count) { + run_loop->Quit(); + } + }, + count, run_loop, result), + base::SequencedTaskRunnerHandle::Get()); + }, + &count, &run_loop, n, result)); + } + // Wait for it all to finish and compare the results. + run_loop.Run(); + EXPECT_THAT(actual_fibo_results, ::testing::Eq(expected_fibo_results)); +} + +// This test runs a series of actions returning non-primitive object as a result +// (Status). +TEST_F(TaskRunner, ActionsWithStatus) { + class ActionsWithStatusContext : public TaskRunnerContext<Status> { + public: + ActionsWithStatusContext( + const std::vector<Status>& vector, + base::OnceCallback<void(Status)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : TaskRunnerContext<Status>(std::move(callback), + std::move(task_runner)), + vector_(vector) {} + + private: + void Pick(size_t index) { + CheckOnValidSequence(); + if (index < vector_.size()) { + if (vector_[index].ok()) { + Schedule(&ActionsWithStatusContext::Pick, base::Unretained(this), + index + 1); + return; + } + Response(vector_[index]); + return; + } + Response(Status(error::OUT_OF_RANGE, "All statuses are OK")); + } + + void OnStart() override { Pick(0); } + + const std::vector<Status> vector_; + }; + + Status result(error::UNKNOWN, "Not yet set"); + base::RunLoop run_loop; + Start<ActionsWithStatusContext>( + std::vector<Status>({Status::StatusOK(), Status::StatusOK(), + Status::StatusOK(), + Status(error::CANCELLED, "Cancelled"), + Status::StatusOK(), Status::StatusOK()}), + base::BindOnce( + [](base::RunLoop* run_loop, Status* result, Status res) { + *result = res; + run_loop->Quit(); + }, + &run_loop, &result), + base::SequencedTaskRunnerHandle::Get()); + run_loop.Run(); + EXPECT_EQ(result, Status(error::CANCELLED, "Cancelled")); +} + +// This test runs a series of actions returning non-primitive non-copyable +// object as a result (StatusOr<std::unique_ptr<...>>). +TEST_F(TaskRunner, ActionsWithStatusOrPtr) { + class WrappedValue { + public: + explicit WrappedValue(int value) : value_(value) {} + ~WrappedValue() = default; + + WrappedValue(const WrappedValue& other) = delete; + WrappedValue& operator=(const WrappedValue& other) = delete; + + int value() const { return value_; } + + private: + const int value_; + }; + using StatusOrPtr = StatusOr<std::unique_ptr<WrappedValue>>; + class ActionsWithStatusOrContext : public TaskRunnerContext<StatusOrPtr> { + public: + ActionsWithStatusOrContext( + std::vector<StatusOrPtr>* vector, + base::OnceCallback<void(StatusOrPtr)> callback, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : TaskRunnerContext<StatusOrPtr>(std::move(callback), + std::move(task_runner)), + vector_(std::move(vector)) {} + + private: + void Pick(size_t index) { + CheckOnValidSequence(); + if (index < vector_->size()) { + if (!vector_->at(index).ok()) { + Schedule(&ActionsWithStatusOrContext::Pick, base::Unretained(this), + index + 1); + return; + } + Response(std::move(vector_->at(index))); + return; + } + Response(Status(error::OUT_OF_RANGE, "All statuses are OK")); + } + + void OnStart() override { Pick(0); } + + std::vector<StatusOrPtr>* const vector_; + }; + + const int kI = 0; + std::vector<StatusOrPtr> vector; + vector.emplace_back(Status(error::CANCELLED, "Cancelled")); + vector.emplace_back(Status(error::CANCELLED, "Cancelled")); + vector.emplace_back(Status(error::CANCELLED, "Cancelled")); + vector.emplace_back(Status(error::CANCELLED, "Cancelled")); + vector.emplace_back(Status(error::CANCELLED, "Cancelled")); + vector.emplace_back(std::make_unique<WrappedValue>(kI)); + StatusOrPtr result; + base::RunLoop run_loop; + Start<ActionsWithStatusOrContext>( + &vector, + base::BindOnce( + [](base::RunLoop* run_loop, StatusOrPtr* result, StatusOrPtr res) { + *result = std::move(res); + run_loop->Quit(); + }, + &run_loop, &result), + base::SequencedTaskRunnerHandle::Get()); + run_loop.Run(); + EXPECT_TRUE(result.ok()) << result.status(); + EXPECT_EQ(result.ValueOrDie()->value(), kI); +} + +} // namespace +} // namespace reporting |