diff options
| author | John Dickinson <me@not.mn> | 2015-04-14 08:10:41 -0700 |
|---|---|---|
| committer | John Dickinson <me@not.mn> | 2015-04-14 08:57:15 -0700 |
| commit | e910f7e07d05dd2c6ada939d5704c3d4944c24b0 (patch) | |
| tree | 250b33237ccbc07bf4ef5895439a0e8cac2fec11 | |
| parent | dd9d97458ea007024220a78dba8dd663e8b425d7 (diff) | |
| parent | 8f5d4d24557887b4691fc219cefbc30e478bf7ed (diff) | |
| download | swift-e910f7e07d05dd2c6ada939d5704c3d4944c24b0.tar.gz | |
Merge EC feature into master
Co-Authored-By: Alistair Coles <alistair.coles@hp.com>
Co-Authored-By: Thiago da Silva <thiago@redhat.com>
Co-Authored-By: John Dickinson <me@not.mn>
Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Co-Authored-By: Tushar Gohad <tushar.gohad@intel.com>
Co-Authored-By: Paul Luse <paul.e.luse@intel.com>
Co-Authored-By: Samuel Merritt <sam@swiftstack.com>
Co-Authored-By: Christian Schwede <christian.schwede@enovance.com>
Co-Authored-By: Yuan Zhou <yuan.zhou@intel.com>
Change-Id: I002787f558781bd4d884129b127bc9f108ea9ec4
91 files changed, 17219 insertions, 2922 deletions
diff --git a/bin/swift-object-reconstructor b/bin/swift-object-reconstructor new file mode 100755 index 000000000..ee4c5d643 --- /dev/null +++ b/bin/swift-object-reconstructor @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.obj.reconstructor import ObjectReconstructor +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon +from optparse import OptionParser + +if __name__ == '__main__': + parser = OptionParser("%prog CONFIG [options]") + parser.add_option('-d', '--devices', + help='Reconstruct only given devices. ' + 'Comma-separated list') + parser.add_option('-p', '--partitions', + help='Reconstruct only given partitions. ' + 'Comma-separated list') + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(ObjectReconstructor, conf_file, **options) diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index a6bb69975..93408cf7a 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -270,6 +270,10 @@ If you need to use an HTTP Proxy, set it here; defaults to no proxy. Will audit, at most, each container once per interval. The default is 300 seconds. .IP \fBcontainer_time\fR Maximum amount of time to spend syncing each container per pass. The default is 60 seconds. +.IP \fBrequest_retries\fR +Server errors from requests will be retried by default. +.IP \fBinternal_client_conf_path\fR +Internal client config file path. .RE .PD diff --git a/doc/saio/bin/remakerings b/doc/saio/bin/remakerings index e95915953..1452cea73 100755 --- a/doc/saio/bin/remakerings +++ b/doc/saio/bin/remakerings @@ -16,6 +16,16 @@ swift-ring-builder object-1.builder add r1z2-127.0.0.1:6020/sdb2 1 swift-ring-builder object-1.builder add r1z3-127.0.0.1:6030/sdb3 1 swift-ring-builder object-1.builder add r1z4-127.0.0.1:6040/sdb4 1 swift-ring-builder object-1.builder rebalance +swift-ring-builder object-2.builder create 10 6 1 +swift-ring-builder object-2.builder add r1z1-127.0.0.1:6010/sdb1 1 +swift-ring-builder object-2.builder add r1z1-127.0.0.1:6010/sdb5 1 +swift-ring-builder object-2.builder add r1z2-127.0.0.1:6020/sdb2 1 +swift-ring-builder object-2.builder add r1z2-127.0.0.1:6020/sdb6 1 +swift-ring-builder object-2.builder add r1z3-127.0.0.1:6030/sdb3 1 +swift-ring-builder object-2.builder add r1z3-127.0.0.1:6030/sdb7 1 +swift-ring-builder object-2.builder add r1z4-127.0.0.1:6040/sdb4 1 +swift-ring-builder object-2.builder add r1z4-127.0.0.1:6040/sdb8 1 +swift-ring-builder object-2.builder rebalance swift-ring-builder container.builder create 10 3 1 swift-ring-builder container.builder add r1z1-127.0.0.1:6011/sdb1 1 swift-ring-builder container.builder add r1z2-127.0.0.1:6021/sdb2 1 diff --git a/doc/saio/bin/resetswift b/doc/saio/bin/resetswift index dd2692f7d..c7c9d9eae 100755 --- a/doc/saio/bin/resetswift +++ b/doc/saio/bin/resetswift @@ -9,7 +9,10 @@ sudo mkfs.xfs -f ${SAIO_BLOCK_DEVICE:-/dev/sdb1} sudo mount /mnt/sdb1 sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 sudo chown ${USER}:${USER} /mnt/sdb1/* -mkdir -p /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 +mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \ + /srv/2/node/sdb2 /srv/2/node/sdb6 \ + /srv/3/node/sdb3 /srv/3/node/sdb7 \ + /srv/4/node/sdb4 /srv/4/node/sdb8 sudo rm -f /var/log/debug /var/log/messages /var/log/rsyncd.log /var/log/syslog find /var/cache/swift* -type f -name *.recon -exec rm -f {} \; # On Fedora use "systemctl restart <service>" diff --git a/doc/saio/swift/object-server/1.conf b/doc/saio/swift/object-server/1.conf index c0300ee55..178e3fcba 100644 --- a/doc/saio/swift/object-server/1.conf +++ b/doc/saio/swift/object-server/1.conf @@ -22,6 +22,8 @@ use = egg:swift#recon [object-replicator] vm_test_mode = yes +[object-reconstructor] + [object-updater] [object-auditor] diff --git a/doc/saio/swift/object-server/2.conf b/doc/saio/swift/object-server/2.conf index 71d373a48..6b611ca25 100644 --- a/doc/saio/swift/object-server/2.conf +++ b/doc/saio/swift/object-server/2.conf @@ -22,6 +22,8 @@ use = egg:swift#recon [object-replicator] vm_test_mode = yes +[object-reconstructor] + [object-updater] [object-auditor] diff --git a/doc/saio/swift/object-server/3.conf b/doc/saio/swift/object-server/3.conf index 4c103b304..735259231 100644 --- a/doc/saio/swift/object-server/3.conf +++ b/doc/saio/swift/object-server/3.conf @@ -22,6 +22,8 @@ use = egg:swift#recon [object-replicator] vm_test_mode = yes +[object-reconstructor] + [object-updater] [object-auditor] diff --git a/doc/saio/swift/object-server/4.conf b/doc/saio/swift/object-server/4.conf index c51d12215..be1211047 100644 --- a/doc/saio/swift/object-server/4.conf +++ b/doc/saio/swift/object-server/4.conf @@ -22,6 +22,8 @@ use = egg:swift#recon [object-replicator] vm_test_mode = yes +[object-reconstructor] + [object-updater] [object-auditor] diff --git a/doc/saio/swift/swift.conf b/doc/saio/swift/swift.conf index 4d8b014e8..25e100264 100644 --- a/doc/saio/swift/swift.conf +++ b/doc/saio/swift/swift.conf @@ -5,7 +5,16 @@ swift_hash_path_suffix = changeme [storage-policy:0] name = gold +policy_type = replication default = yes [storage-policy:1] name = silver +policy_type = replication + +[storage-policy:2] +name = ec42 +policy_type = erasure_coding +ec_type = jerasure_rs_vand +ec_num_data_fragments = 4 +ec_num_parity_fragments = 2 diff --git a/doc/source/associated_projects.rst b/doc/source/associated_projects.rst index 72ed9c016..c0f8cf7e5 100644 --- a/doc/source/associated_projects.rst +++ b/doc/source/associated_projects.rst @@ -104,5 +104,7 @@ Other * `Swiftsync <https://github.com/stackforge/swiftsync>`_ - A massive syncer between two swift clusters. * `Django Swiftbrowser <https://github.com/cschwede/django-swiftbrowser>`_ - Simple Django web app to access Openstack Swift. * `Swift-account-stats <https://github.com/enovance/swift-account-stats>`_ - Swift-account-stats is a tool to report statistics on Swift usage at tenant and global levels. +* `PyECLib <https://bitbucket.org/kmgreen2/pyeclib>`_ - High Level Erasure Code library used by Swift +* `liberasurecode <http://www.bytebucket.org/tsg-/liberasurecode>`_ - Low Level Erasure Code library used by PyECLib * `Swift Browser <https://github.com/zerovm/swift-browser>`_ - JavaScript interface for Swift * `swift-ui <https://github.com/fanatic/swift-ui>`_ - OpenStack Swift web browser diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 338b1420c..3bd94872d 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -87,8 +87,11 @@ another device when creating the VM, and follow these instructions: sudo chown ${USER}:${USER} /mnt/sdb1/* sudo mkdir /srv for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done - sudo mkdir -p /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 \ - /srv/4/node/sdb4 /var/run/swift + sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \ + /srv/2/node/sdb2 /srv/2/node/sdb6 \ + /srv/3/node/sdb3 /srv/3/node/sdb7 \ + /srv/4/node/sdb4 /srv/4/node/sdb8 \ + /var/run/swift sudo chown -R ${USER}:${USER} /var/run/swift # **Make sure to include the trailing slash after /srv/$x/** for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done @@ -124,7 +127,11 @@ these instructions: sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 sudo chown ${USER}:${USER} /mnt/sdb1/* for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done - sudo mkdir -p /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift + sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \ + /srv/2/node/sdb2 /srv/2/node/sdb6 \ + /srv/3/node/sdb3 /srv/3/node/sdb7 \ + /srv/4/node/sdb4 /srv/4/node/sdb8 \ + /var/run/swift sudo chown -R ${USER}:${USER} /var/run/swift # **Make sure to include the trailing slash after /srv/$x/** for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done @@ -402,7 +409,7 @@ Setting up scripts for running Swift #. Copy the SAIO scripts for resetting the environment:: - cd $HOME/swift/doc; cp -r saio/bin $HOME/bin; cd - + cd $HOME/swift/doc; cp saio/bin/* $HOME/bin; cd - chmod +x $HOME/bin/* #. Edit the ``$HOME/bin/resetswift`` script @@ -455,30 +462,41 @@ Setting up scripts for running Swift .. literalinclude:: /../saio/bin/remakerings - You can expect the output from this command to produce the following (note - that 2 object rings are created in order to test storage policies in the - SAIO environment however they map to the same nodes):: + You can expect the output from this command to produce the following. Note + that 3 object rings are created in order to test storage policies and EC in + the SAIO environment. The EC ring is the only one with all 8 devices. + There are also two replication rings, one for 3x replication and another + for 2x replication, but those rings only use 4 devices:: Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0 Device d1r1z2-127.0.0.1:6020R127.0.0.1:6020/sdb2_"" with 1.0 weight got id 1 Device d2r1z3-127.0.0.1:6030R127.0.0.1:6030/sdb3_"" with 1.0 weight got id 2 Device d3r1z4-127.0.0.1:6040R127.0.0.1:6040/sdb4_"" with 1.0 weight got id 3 - Reassigned 1024 (100.00%) partitions. Balance is now 0.00. + Reassigned 1024 (100.00%) partitions. Balance is now 0.00. Dispersion is now 0.00 Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0 Device d1r1z2-127.0.0.1:6020R127.0.0.1:6020/sdb2_"" with 1.0 weight got id 1 Device d2r1z3-127.0.0.1:6030R127.0.0.1:6030/sdb3_"" with 1.0 weight got id 2 Device d3r1z4-127.0.0.1:6040R127.0.0.1:6040/sdb4_"" with 1.0 weight got id 3 - Reassigned 1024 (100.00%) partitions. Balance is now 0.00. + Reassigned 1024 (100.00%) partitions. Balance is now 0.00. Dispersion is now 0.00 + Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0 + Device d1r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb5_"" with 1.0 weight got id 1 + Device d2r1z2-127.0.0.1:6020R127.0.0.1:6020/sdb2_"" with 1.0 weight got id 2 + Device d3r1z2-127.0.0.1:6020R127.0.0.1:6020/sdb6_"" with 1.0 weight got id 3 + Device d4r1z3-127.0.0.1:6030R127.0.0.1:6030/sdb3_"" with 1.0 weight got id 4 + Device d5r1z3-127.0.0.1:6030R127.0.0.1:6030/sdb7_"" with 1.0 weight got id 5 + Device d6r1z4-127.0.0.1:6040R127.0.0.1:6040/sdb4_"" with 1.0 weight got id 6 + Device d7r1z4-127.0.0.1:6040R127.0.0.1:6040/sdb8_"" with 1.0 weight got id 7 + Reassigned 1024 (100.00%) partitions. Balance is now 0.00. Dispersion is now 0.00 Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0 Device d1r1z2-127.0.0.1:6021R127.0.0.1:6021/sdb2_"" with 1.0 weight got id 1 Device d2r1z3-127.0.0.1:6031R127.0.0.1:6031/sdb3_"" with 1.0 weight got id 2 Device d3r1z4-127.0.0.1:6041R127.0.0.1:6041/sdb4_"" with 1.0 weight got id 3 - Reassigned 1024 (100.00%) partitions. Balance is now 0.00. + Reassigned 1024 (100.00%) partitions. Balance is now 0.00. Dispersion is now 0.00 Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0 Device d1r1z2-127.0.0.1:6022R127.0.0.1:6022/sdb2_"" with 1.0 weight got id 1 Device d2r1z3-127.0.0.1:6032R127.0.0.1:6032/sdb3_"" with 1.0 weight got id 2 Device d3r1z4-127.0.0.1:6042R127.0.0.1:6042/sdb4_"" with 1.0 weight got id 3 - Reassigned 1024 (100.00%) partitions. Balance is now 0.00. + Reassigned 1024 (100.00%) partitions. Balance is now 0.00. Dispersion is now 0.00 #. Read more about Storage Policies and your SAIO :doc:`policies_saio` diff --git a/doc/source/images/ec_overview.png b/doc/source/images/ec_overview.png Binary files differnew file mode 100755 index 000000000..d44a10317 --- /dev/null +++ b/doc/source/images/ec_overview.png diff --git a/doc/source/index.rst b/doc/source/index.rst index 630e6bd70..45ee1fd0e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -56,6 +56,7 @@ Overview and Concepts overview_expiring_objects cors crossdomain + overview_erasure_code overview_backing_store associated_projects diff --git a/doc/source/overview_architecture.rst b/doc/source/overview_architecture.rst index b8c9a32f7..1f3452a55 100644 --- a/doc/source/overview_architecture.rst +++ b/doc/source/overview_architecture.rst @@ -11,7 +11,10 @@ Proxy Server The Proxy Server is responsible for tying together the rest of the Swift architecture. For each request, it will look up the location of the account, container, or object in the ring (see below) and route the request accordingly. -The public API is also exposed through the Proxy Server. +For Erasure Code type policies, the Proxy Server is also responsible for +encoding and decoding object data. See :doc:`overview_erasure_code` for +complete information on Erasure Code suport. The public API is also exposed +through the Proxy Server. A large number of failures are also handled in the Proxy Server. For example, if a server is unavailable for an object PUT, it will ask the @@ -87,7 +90,8 @@ implementing a particular differentiation. For example, one might have the default policy with 3x replication, and create a second policy which, when applied to new containers only uses 2x replication. Another might add SSDs to a set of storage nodes and create a performance tier -storage policy for certain containers to have their objects stored there. +storage policy for certain containers to have their objects stored there. Yet +another might be the use of Erasure Coding to define a cold-storage tier. This mapping is then exposed on a per-container basis, where each container can be assigned a specific storage policy when it is created, which remains in @@ -156,6 +160,15 @@ item (object, container, or account) is deleted, a tombstone is set as the latest version of the item. The replicator will see the tombstone and ensure that the item is removed from the entire system. +-------------- +Reconstruction +-------------- + +The reconstructor is used by Erasure Code policies and is analogous to the +replicator for Replication type policies. See :doc:`overview_erasure_code` +for complete information on both Erasure Code support as well as the +reconstructor. + -------- Updaters -------- diff --git a/doc/source/overview_erasure_code.rst b/doc/source/overview_erasure_code.rst new file mode 100755 index 000000000..9927e2ace --- /dev/null +++ b/doc/source/overview_erasure_code.rst @@ -0,0 +1,672 @@ +==================== +Erasure Code Support +==================== + + +-------------------------- +Beta: Not production ready +-------------------------- +The erasure code support in Swift is considered "beta" at this point. +Most major functionality is included, but it has not been tested or validated +at large scale. This feature relies on ssync for durability. Deployers are +urged to do extensive testing and not deploy production data using an +erasure code storage policy. + +If any bugs are found during testing, please report them to +https://bugs.launchpad.net/swift + + +------------------------------- +History and Theory of Operation +------------------------------- + +There's a lot of good material out there on Erasure Code (EC) theory, this short +introduction is just meant to provide some basic context to help the reader +better understand the implementation in Swift. + +Erasure Coding for storage applications grew out of Coding Theory as far back as +the 1960s with the Reed-Solomon codes. These codes have been used for years in +applications ranging from CDs to DVDs to general communications and, yes, even +in the space program starting with Voyager! The basic idea is that some amount +of data is broken up into smaller pieces called fragments and coded in such a +way that it can be transmitted with the ability to tolerate the loss of some +number of the coded fragments. That's where the word "erasure" comes in, if you +transmit 14 fragments and only 13 are received then one of them is said to be +"erased". The word "erasure" provides an important distinction with EC; it +isn't about detecting errors, it's about dealing with failures. Another +important element of EC is that the number of erasures that can be tolerated can +be adjusted to meet the needs of the application. + +At a high level EC works by using a specific scheme to break up a single data +buffer into several smaller data buffers then, depending on the scheme, +performing some encoding operation on that data in order to generate additional +information. So you end up with more data than you started with and that extra +data is often called "parity". Note that there are many, many different +encoding techniques that vary both in how they organize and manipulate the data +as well by what means they use to calculate parity. For example, one scheme +might rely on `Galois Field Arithmetic <http://www.ssrc.ucsc.edu/Papers/plank- +fast13.pdf>`_ while others may work with only XOR. The number of variations and +details about their differences are well beyond the scope of this introduction, +but we will talk more about a few of them when we get into the implementation of +EC in Swift. + +-------------------------------- +Overview of EC Support in Swift +-------------------------------- + +First and foremost, from an application perspective EC support is totally +transparent. There are no EC related external API; a container is simply created +using a Storage Policy defined to use EC and then interaction with the cluster +is the same as any other durability policy. + +EC is implemented in Swift as a Storage Policy, see :doc:`overview_policies` for +complete details on Storage Policies. Because support is implemented as a +Storage Policy, all of the storage devices associated with your cluster's EC +capability can be isolated. It is entirely possible to share devices between +storage policies, but for EC it may make more sense to not only use separate +devices but possibly even entire nodes dedicated for EC. + +Which direction one chooses depends on why the EC policy is being deployed. If, +for example, there is a production replication policy in place already and the +goal is to add a cold storage tier such that the existing nodes performing +replication are impacted as little as possible, adding a new set of nodes +dedicated to EC might make the most sense but also incurs the most cost. On the +other hand, if EC is being added as a capability to provide additional +durability for a specific set of applications and the existing infrastructure is +well suited for EC (sufficient number of nodes, zones for the EC scheme that is +chosen) then leveraging the existing infrastructure such that the EC ring shares +nodes with the replication ring makes the most sense. These are some of the +main considerations: + +* Layout of existing infrastructure. +* Cost of adding dedicated EC nodes (or just dedicated EC devices). +* Intended usage model(s). + +The Swift code base does not include any of the algorithms necessary to perform +the actual encoding and decoding of data; that is left to external libraries. +The Storage Policies architecture is leveraged to enable EC on a per container +basis -- the object rings are still used to determine the placement of EC data +fragments. Although there are several code paths that are unique to an operation +associated with an EC policy, an external dependency to an Erasure Code library +is what Swift counts on to perform the low level EC functions. The use of an +external library allows for maximum flexibility as there are a significant +number of options out there, each with its owns pros and cons that can vary +greatly from one use case to another. + +--------------------------------------- +PyECLib: External Erasure Code Library +--------------------------------------- + +PyECLib is a Python Erasure Coding Library originally designed and written as +part of the effort to add EC support to the Swift project, however it is an +independent project. The library provides a well-defined and simple Python +interface and internally implements a plug-in architecture allowing it to take +advantage of many well-known C libraries such as: + +* Jerasure and GFComplete at http://jerasure.org. +* Intel(R) ISA-L at http://01.org/intel%C2%AE-storage-acceleration-library-open-source-version. +* Or write your own! + +PyECLib uses a C based library called liberasurecode to implement the plug in +infrastructure; liberasure code is available at: + +* liberasurecode: https://bitbucket.org/tsg-/liberasurecode + +PyECLib itself therefore allows for not only choice but further extensibility as +well. PyECLib also comes with a handy utility to help determine the best +algorithm to use based on the equipment that will be used (processors and server +configurations may vary in performance per algorithm). More on this will be +covered in the configuration section. PyECLib is included as a Swift +requirement. + +For complete details see `PyECLib <https://bitbucket.org/kmgreen2/pyeclib>`_ + +------------------------------ +Storing and Retrieving Objects +------------------------------ + +We will discuss the details of how PUT and GET work in the "Under the Hood" +section later on. The key point here is that all of the erasure code work goes +on behind the scenes; this summary is a high level information overview only. + +The PUT flow looks like this: + +#. The proxy server streams in an object and buffers up "a segment" of data + (size is configurable). +#. The proxy server calls on PyECLib to encode the data into smaller fragments. +#. The proxy streams the encoded fragments out to the storage nodes based on + ring locations. +#. Repeat until the client is done sending data. +#. The client is notified of completion when a quorum is met. + +The GET flow looks like this: + +#. The proxy server makes simultaneous requests to participating nodes. +#. As soon as the proxy has the fragments it needs, it calls on PyECLib to + decode the data. +#. The proxy streams the decoded data it has back to the client. +#. Repeat until the proxy is done sending data back to the client. + +It may sound like, from this high level overview, that using EC is going to +cause an explosion in the number of actual files stored in each node's local +file system. Although it is true that more files will be stored (because an +object is broken into pieces), the implementation works to minimize this where +possible, more details are available in the Under the Hood section. + +------------- +Handoff Nodes +------------- + +In EC policies, similarly to replication, handoff nodes are a set of storage +nodes used to augment the list of primary nodes responsible for storing an +erasure coded object. These handoff nodes are used in the event that one or more +of the primaries are unavailable. Handoff nodes are still selected with an +attempt to achieve maximum separation of the data being placed. + +-------------- +Reconstruction +-------------- + +For an EC policy, reconstruction is analogous to the process of replication for +a replication type policy -- essentially "the reconstructor" replaces "the +replicator" for EC policy types. The basic framework of reconstruction is very +similar to that of replication with a few notable exceptions: + +* Because EC does not actually replicate partitions, it needs to operate at a + finer granularity than what is provided with rsync, therefore EC leverages + much of ssync behind the scenes (you do not need to manually configure ssync). +* Once a pair of nodes has determined the need to replace a missing object + fragment, instead of pushing over a copy like replication would do, the + reconstructor has to read in enough surviving fragments from other nodes and + perform a local reconstruction before it has the correct data to push to the + other node. +* A reconstructor does not talk to all other reconstructors in the set of nodes + responsible for an EC partition, this would be far too chatty, instead each + reconstructor is responsible for sync'ing with the partition's closest two + neighbors (closest meaning left and right on the ring). + +.. note:: + + EC work (encode and decode) takes place both on the proxy nodes, for PUT/GET + operations, as well as on the storage nodes for reconstruction. As with + replication, reconstruction can be the result of rebalancing, bit-rot, drive + failure or reverting data from a hand-off node back to its primary. + +-------------------------- +Performance Considerations +-------------------------- + +Efforts are underway to characterize performance of various Erasure Code +schemes. One of the main goals of the beta release is to perform this +characterization and encourage others to do so and provide meaningful feedback +to the development community. There are many factors that will affect +performance of EC so it is vital that we have multiple characterization +activities happening. + +In general, EC has different performance characteristics than replicated data. +EC requires substantially more CPU to read and write data, and is more suited +for larger objects that are not frequently accessed (eg backups). + +---------------------------- +Using an Erasure Code Policy +---------------------------- + +To use an EC policy, the administrator simply needs to define an EC policy in +`swift.conf` and create/configure the associated object ring. An example of how +an EC policy can be setup is shown below:: + + [storage-policy:2] + name = ec104 + policy_type = erasure_coding + ec_type = jerasure_rs_vand + ec_num_data_fragments = 10 + ec_num_parity_fragments = 4 + ec_object_segment_size = 1048576 + +Let's take a closer look at each configuration parameter: + +* ``name``: This is a standard storage policy parameter. + See :doc:`overview_policies` for details. +* ``policy_type``: Set this to ``erasure_coding`` to indicate that this is an EC + policy. +* ``ec_type``: Set this value according to the available options in the selected + PyECLib back-end. This specifies the EC scheme that is to be used. For + example the option shown here selects Vandermonde Reed-Solomon encoding while + an option of ``flat_xor_hd_3`` would select Flat-XOR based HD combination + codes. See the `PyECLib <https://bitbucket.org/kmgreen2/pyeclib>`_ page for + full details. +* ``ec_num_data_fragments``: The total number of fragments that will be + comprised of data. +* ``ec_num_parity_fragments``: The total number of fragments that will be + comprised of parity. +* ``ec_object_segment_size``: The amount of data that will be buffered up before + feeding a segment into the encoder/decoder. The default value is 1048576. + +When PyECLib encodes an object, it will break it into N fragments. However, what +is important during configuration, is how many of those are data and how many +are parity. So in the example above, PyECLib will actually break an object in +14 different fragments, 10 of them will be made up of actual object data and 4 +of them will be made of parity data (calculations depending on ec_type). + +When deciding which devices to use in the EC policy's object ring, be sure to +carefully consider the performance impacts. Running some performance +benchmarking in a test environment for your configuration is highly recommended +before deployment. Once you have configured your EC policy in `swift.conf` and +created your object ring, your application is ready to start using EC simply by +creating a container with the specified policy name and interacting as usual. + +.. note:: + + It's important to note that once you have deployed a policy and have created + objects with that policy, these configurations options cannot be changed. In + case a change in the configuration is desired, you must create a new policy + and migrate the data to a new container. + +Migrating Between Policies +-------------------------- + +A common usage of EC is to migrate less commonly accessed data from a more +expensive but lower latency policy such as replication. When an application +determines that it wants to move data from a replication policy to an EC policy, +it simply needs to move the data from the replicated container to an EC +container that was created with the target durability policy. + +Region Support +-------------- + +For at least the initial version of EC, it is not recommended that an EC scheme +span beyond a single region, neither performance nor functional validation has +be been done in such a configuration. + +-------------- +Under the Hood +-------------- + +Now that we've explained a little about EC support in Swift and how to +configure/use it, let's explore how EC fits in at the nuts-n-bolts level. + +Terminology +----------- + +The term 'fragment' has been used already to describe the output of the EC +process (a series of fragments) however we need to define some other key terms +here before going any deeper. Without paying special attention to using the +correct terms consistently, it is very easy to get confused in a hurry! + +* **chunk**: HTTP chunks received over wire (term not used to describe any EC + specific operation). +* **segment**: Not to be confused with SLO/DLO use of the word, in EC we call a + segment a series of consecutive HTTP chunks buffered up before performing an + EC operation. +* **fragment**: Data and parity 'fragments' are generated when erasure coding + transformation is applied to a segment. +* **EC archive**: A concatenation of EC fragments; to a storage node this looks + like an object. +* **ec_ndata**: Number of EC data fragments. +* **ec_nparity**: Number of EC parity fragments. + +Middleware +---------- + +Middleware remains unchanged. For most middleware (e.g., SLO/DLO) the fact that +the proxy is fragmenting incoming objects is transparent. For list endpoints, +however, it is a bit different. A caller of list endpoints will get back the +locations of all of the fragments. The caller will be unable to re-assemble the +original object with this information, however the node locations may still +prove to be useful information for some applications. + +On Disk Storage +--------------- + +EC archives are stored on disk in their respective objects-N directory based on +their policy index. See :doc:`overview_policies` for details on per policy +directory information. + +The actual names on disk of EC archives also have one additional piece of data +encoded in the filename, the fragment archive index. + +Each storage policy now must include a transformation function that diskfile +will use to build the filename to store on disk. The functions are implemented +in the diskfile module as policy specific sub classes ``DiskFileManager``. + +This is required for a few reasons. For one, it allows us to store fragment +archives of different indexes on the same storage node which is not typical +however it is possible in many circumstances. Without unique filenames for the +different EC archive files in a set, we would be at risk of overwriting one +archive of index n with another of index m in some scenarios. + +The transformation function for the replication policy is simply a NOP. For +reconstruction, the index is appended to the filename just before the .data +extension. An example filename for a fragment archive storing the 5th fragment +would like this this:: + + 1418673556.92690#5.data + +An additional file is also included for Erasure Code policies called the +``.durable`` file. Its meaning will be covered in detail later, however, its on- +disk format does not require the name transformation function that was just +covered. The .durable for the example above would simply look like this:: + + 1418673556.92690.durable + +And it would be found alongside every fragment specific .data file following a +100% successful PUT operation. + +Proxy Server +------------ + +High Level +========== + +The Proxy Server handles Erasure Coding in a different manner than replication, +therefore there are several code paths unique to EC policies either though sub +classing or simple conditionals. Taking a closer look at the PUT and the GET +paths will help make this clearer. But first, a high level overview of how an +object flows through the system: + +.. image:: images/ec_overview.png + +Note how: + +* Incoming objects are buffered into segments at the proxy. +* Segments are erasure coded into fragments at the proxy. +* The proxy stripes fragments across participating nodes such that the on-disk + stored files that we call a fragment archive is appended with each new + fragment. + +This scheme makes it possible to minimize the number of on-disk files given our +segmenting and fragmenting. + +Multi_Phase Conversation +======================== + +Multi-part MIME document support is used to allow the proxy to engage in a +handshake conversation with the storage node for processing PUT requests. This +is required for a few different reasons. + +#. From the perspective of the storage node, a fragment archive is really just + another object, we need a mechanism to send down the original object etag + after all fragment archives have landed. +#. Without introducing strong consistency semantics, the proxy needs a mechanism + to know when a quorum of fragment archives have actually made it to disk + before it can inform the client of a successful PUT. + +MIME supports a conversation between the proxy and the storage nodes for every +PUT. This provides us with the ability to handle a PUT in one connection and +assure that we have the essence of a 2 phase commit, basically having the proxy +communicate back to the storage nodes once it has confirmation that all fragment +archives in the set have been committed. Note that we still require a quorum of +data elements of the conversation to complete before signaling status to the +client but we can relax that requirement for the commit phase such that only 2 +confirmations to that phase of the conversation are required for success as the +reconstructor will assure propagation of markers that indicate data durability. + +This provides the storage node with a cheap indicator of the last known durable +set of fragment archives for a given object on a successful durable PUT, this is +known as the ``.durable`` file. The presence of a ``.durable`` file means, to +the object server, `there is a set of ts.data files that are durable at +timestamp ts.` Note that the completion of the commit phase of the conversation +is also a signal for the object server to go ahead and immediately delete older +timestamp files for this object. This is critical as we do not want to delete +the older object until the storage node has confirmation from the proxy, via the +multi-phase conversation, that the other nodes have landed enough for a quorum. + +The basic flow looks like this: + + * The Proxy Server erasure codes and streams the object fragments + (ec_ndata + ec_nparity) to the storage nodes. + * The storage nodes store objects as EC archives and upon finishing object + data/metadata write, send a 1st-phase response to proxy. + * Upon quorum of storage nodes responses, the proxy initiates 2nd-phase by + sending commit confirmations to object servers. + * Upon receipt of commit message, object servers store a 0-byte data file as + `<timestamp>.durable` indicating successful PUT, and send a final response to + the proxy server. + * The proxy waits for a minimal number of two object servers to respond with a + success (2xx) status before responding to the client with a successful + status. In this particular case it was decided that two responses was + the mininum amount to know that the file would be propagated in case of + failure from other others and because a greater number would potentially + mean more latency, which should be avoided if possible. + +Here is a high level example of what the conversation looks like:: + + proxy: PUT /p/a/c/o + Transfer-Encoding': 'chunked' + Expect': '100-continue' + X-Backend-Obj-Multiphase-Commit: yes + obj: 100 Continue + X-Obj-Multiphase-Commit: yes + proxy: --MIMEboundary + X-Document: object body + <obj_data> + --MIMEboundary + X-Document: object metadata + Content-MD5: <footer_meta_cksum> + <footer_meta> + --MIMEboundary + <object server writes data, metadata> + obj: 100 Continue + <quorum> + proxy: X-Document: put commit + commit_confirmation + --MIMEboundary-- + <object server writes ts.durable state> + obj: 20x + <proxy waits to receive >=2 2xx responses> + proxy: 2xx -> client + +A few key points on the .durable file: + +* The .durable file means \"the matching .data file for this has sufficient + fragment archives somewhere, committed, to reconstruct the object\". +* The Proxy Server will never have knowledge, either on GET or HEAD, of the + existence of a .data file on an object server if it does not have a matching + .durable file. +* The object server will never return a .data that does not have a matching + .durable. +* When a proxy does a GET, it will only receive fragment archives that have + enough present somewhere to be reconstructed. + +Partial PUT Failures +==================== + +A partial PUT failure has a few different modes. In one scenario the Proxy +Server is alive through the entire PUT conversation. This is a very +straightforward case. The client will receive a good response if and only if a +quorum of fragment archives were successfully landed on their storage nodes. In +this case the Reconstructor will discover the missing fragment archives, perform +a reconstruction and deliver fragment archives and their matching .durable files +to the nodes. + +The more interesting case is what happens if the proxy dies in the middle of a +conversation. If it turns out that a quorum had been met and the commit phase +of the conversation finished, its as simple as the previous case in that the +reconstructor will repair things. However, if the commit didn't get a change to +happen then some number of the storage nodes have .data files on them (fragment +archives) but none of them knows whether there are enough elsewhere for the +entire object to be reconstructed. In this case the client will not have +received a 2xx response so there is no issue there, however, it is left to the +storage nodes to clean up the stale fragment archives. Work is ongoing in this +area to enable the proxy to play a role in reviving these fragment archives, +however, for the current release, a proxy failure after the start of a +conversation but before the commit message will simply result in a PUT failure. + +GET +=== + +The GET for EC is different enough from replication that subclassing the +`BaseObjectController` to the `ECObjectController` enables an efficient way to +implement the high level steps described earlier: + +#. The proxy server makes simultaneous requests to participating nodes. +#. As soon as the proxy has the fragments it needs, it calls on PyECLib to + decode the data. +#. The proxy streams the decoded data it has back to the client. +#. Repeat until the proxy is done sending data back to the client. + +The GET path will attempt to contact all nodes participating in the EC scheme, +if not enough primaries respond then handoffs will be contacted just as with +replication. Etag and content length headers are updated for the client +response following reconstruction as the individual fragment archives metadata +is valid only for that fragment archive. + +Object Server +------------- + +The Object Server, like the Proxy Server, supports MIME conversations as +described in the proxy section earlier. This includes processing of the commit +message and decoding various sections of the MIME document to extract the footer +which includes things like the entire object etag. + +DiskFile +======== + +Erasure code uses subclassed ``ECDiskFile``, ``ECDiskFileWriter`` and +``ECDiskFileManager`` to impement EC specific handling of on disk files. This +includes things like file name manipulation to include the fragment index in the +filename, determination of valid .data files based on .durable presence, +construction of EC specific hashes.pkl file to include fragment index +information, etc., etc. + +Metadata +-------- + +There are few different categories of metadata that are associated with EC: + +System Metadata: EC has a set of object level system metadata that it +attaches to each of the EC archives. The metadata is for internal use only: + +* ``X-Object-Sysmeta-EC-Etag``: The Etag of the original object. +* ``X-Object-Sysmeta-EC-Content-Length``: The content length of the original + object. +* ``X-Object-Sysmeta-EC-Frag-Index``: The fragment index for the object. +* ``X-Object-Sysmeta-EC-Scheme``: Description of the EC policy used to encode + the object. +* ``X-Object-Sysmeta-EC-Segment-Size``: The segment size used for the object. + +User Metadata: User metadata is unaffected by EC, however, a full copy of the +user metadata is stored with every EC archive. This is required as the +reconstructor needs this information and each reconstructor only communicates +with its closest neighbors on the ring. + +PyECLib Metadata: PyECLib stores a small amount of metadata on a per fragment +basis. This metadata is not documented here as it is opaque to Swift. + +Database Updates +---------------- + +As account and container rings are not associated with a Storage Policy, there +is no change to how these database updates occur when using an EC policy. + +The Reconstructor +----------------- + +The Reconstructor performs analogous functions to the replicator: + +#. Recovery from disk drive failure. +#. Moving data around because of a rebalance. +#. Reverting data back to a primary from a handoff. +#. Recovering fragment archives from bit rot discovered by the auditor. + +However, under the hood it operates quite differently. The following are some +of the key elements in understanding how the reconstructor operates. + +Unlike the replicator, the work that the reconstructor does is not always as +easy to break down into the 2 basic tasks of synchronize or revert (move data +from handoff back to primary) because of the fact that one storage node can +house fragment archives of various indexes and each index really /"belongs/" to +a different node. So, whereas when the replicator is reverting data from a +handoff it has just one node to send its data to, the reconstructor can have +several. Additionally, its not always the case that the processing of a +particular suffix directory means one or the other for the entire directory (as +it does for replication). The scenarios that create these mixed situations can +be pretty complex so we will just focus on what the reconstructor does here and +not a detailed explanation of why. + +Job Construction and Processing +=============================== + +Because of the nature of the work it has to do as described above, the +reconstructor builds jobs for a single job processor. The job itself contains +all of the information needed for the processor to execute the job which may be +a synchronization or a data reversion and there may be a mix of jobs that +perform both of these operations on the same suffix directory. + +Jobs are constructed on a per partition basis and then per fragment index basis. +That is, there will be one job for every fragment index in a partition. +Performing this construction \"up front\" like this helps minimize the +interaction between nodes collecting hashes.pkl information. + +Once a set of jobs for a partition has been constructed, those jobs are sent off +to threads for execution. The single job processor then performs the necessary +actions working closely with ssync to carry out its instructions. For data +reversion, the actual objects themselves are cleaned up via the ssync module and +once that partition's set of jobs is complete, the reconstructor will attempt to +remove the relevant directory structures. + +The scenarios that job construction has to take into account include: + +#. A partition directory with all fragment indexes matching the local node + index. This is the case where everything is where it belongs and we just + need to compare hashes and sync if needed, here we sync with our partners. +#. A partition directory with one local fragment index and mix of others. Here + we need to sync with our partners where fragment indexes matches the + local_id, all others are sync'd with their home nodes and then deleted. +#. A partition directory with no local fragment index and just one or more of + others. Here we sync with just the home nodes for the fragment indexes that + we have and then all the local archives are deleted. This is the basic + handoff reversion case. + +.. note:: + A \"home node\" is the node where the fragment index encoded in the + fragment archive's filename matches the node index of a node in the primary + partition list. + +Node Communication +================== + +The replicators talk to all nodes who have a copy of their object, typically +just 2 other nodes. For EC, having each reconstructor node talk to all nodes +would incur a large amount of overhead as there will typically be a much larger +number of nodes participating in the EC scheme. Therefore, the reconstructor is +built to talk to its adjacent nodes on the ring only. These nodes are typically +referred to as partners. + +Reconstruction +============== + +Reconstruction can be thought of sort of like replication but with an extra step +in the middle. The reconstructor is hard-wired to use ssync to determine what is +missing and desired by the other side. However, before an object is sent over +the wire it needs to be reconstructed from the remaining fragments as the local +fragment is just that - a different fragment index than what the other end is +asking for. + +Thus, there are hooks in ssync for EC based policies. One case would be for +basic reconstruction which, at a high level, looks like this: + +* Determine which nodes need to be contacted to collect other EC archives needed + to perform reconstruction. +* Update the etag and fragment index metadata elements of the newly constructed + fragment archive. +* Establish a connection to the target nodes and give ssync a DiskFileLike class + that it can stream data from. + +The reader in this class gathers fragments from the nodes and uses PyECLib to +reconstruct each segment before yielding data back to ssync. Essentially what +this means is that data is buffered, in memory, on a per segment basis at the +node performing reconstruction and each segment is dynamically reconstructed and +delivered to `ssync_sender` where the `send_put()` method will ship them on +over. The sender is then responsible for deleting the objects as they are sent +in the case of data reversion. + +The Auditor +----------- + +Because the auditor already operates on a per storage policy basis, there are no +specific auditor changes associated with EC. Each EC archive looks like, and is +treated like, a regular object from the perspective of the auditor. Therefore, +if the auditor finds bit-rot in an EC archive, it simply quarantines it and the +reconstructor will take care of the rest just as the replicator does for +replication policies. diff --git a/doc/source/overview_policies.rst b/doc/source/overview_policies.rst index 9cabde6cf..06c7fc79a 100755 --- a/doc/source/overview_policies.rst +++ b/doc/source/overview_policies.rst @@ -8,22 +8,22 @@ feature is implemented throughout the entire code base so it is an important concept in understanding Swift architecture. As described in :doc:`overview_ring`, Swift uses modified hashing rings to -determine where data should reside in the cluster. There is a separate ring -for account databases, container databases, and there is also one object -ring per storage policy. Each object ring behaves exactly the same way -and is maintained in the same manner, but with policies, different devices -can belong to different rings with varying levels of replication. By supporting -multiple object rings, Swift allows the application and/or deployer to -essentially segregate the object storage within a single cluster. There are -many reasons why this might be desirable: - -* Different levels of replication: If a provider wants to offer, for example, - 2x replication and 3x replication but doesn't want to maintain 2 separate clusters, - they would setup a 2x policy and a 3x policy and assign the nodes to their - respective rings. - -* Performance: Just as SSDs can be used as the exclusive members of an account or - database ring, an SSD-only object ring can be created as well and used to +determine where data should reside in the cluster. There is a separate ring for +account databases, container databases, and there is also one object ring per +storage policy. Each object ring behaves exactly the same way and is maintained +in the same manner, but with policies, different devices can belong to different +rings. By supporting multiple object rings, Swift allows the application and/or +deployer to essentially segregate the object storage within a single cluster. +There are many reasons why this might be desirable: + +* Different levels of durability: If a provider wants to offer, for example, + 2x replication and 3x replication but doesn't want to maintain 2 separate + clusters, they would setup a 2x and a 3x replication policy and assign the + nodes to their respective rings. Furthermore, if a provider wanted to offer a + cold storage tier, they could create an erasure coded policy. + +* Performance: Just as SSDs can be used as the exclusive members of an account + or database ring, an SSD-only object ring can be created as well and used to implement a low-latency/high performance policy. * Collecting nodes into group: Different object rings may have different @@ -36,10 +36,12 @@ many reasons why this might be desirable: .. note:: - Today, choosing a different storage policy allows the use of different - object rings, but future policies (such as Erasure Coding) will also - change some of the actual code paths when processing a request. Also note - that Diskfile refers to backend object storage plug-in architecture. + Today, Swift supports two different policy types: Replication and Erasure + Code. Erasure Code policy is currently a beta release and should not be + used in a Production cluster. See :doc:`overview_erasure_code` for details. + + Also note that Diskfile refers to backend object storage plug-in + architecture. See :doc:`development_ondisk_backends` for details. ----------------------- Containers and Policies @@ -61,31 +63,33 @@ Policy-0 is considered the default). We will be covering the difference between default and Policy-0 in the next section. Policies are assigned when a container is created. Once a container has been -assigned a policy, it cannot be changed (unless it is deleted/recreated). The implications -on data placement/movement for large datasets would make this a task best left for -applications to perform. Therefore, if a container has an existing policy of, -for example 3x replication, and one wanted to migrate that data to a policy that specifies -a different replication level, the application would create another container -specifying the other policy name and then simply move the data from one container -to the other. Policies apply on a per container basis allowing for minimal application -awareness; once a container has been created with a specific policy, all objects stored -in it will be done so in accordance with that policy. If a container with a -specific name is deleted (requires the container be empty) a new container may -be created with the same name without any restriction on storage policy -enforced by the deleted container which previously shared the same name. +assigned a policy, it cannot be changed (unless it is deleted/recreated). The +implications on data placement/movement for large datasets would make this a +task best left for applications to perform. Therefore, if a container has an +existing policy of, for example 3x replication, and one wanted to migrate that +data to an Erasure Code policy, the application would create another container +specifying the other policy parameters and then simply move the data from one +container to the other. Policies apply on a per container basis allowing for +minimal application awareness; once a container has been created with a specific +policy, all objects stored in it will be done so in accordance with that policy. +If a container with a specific name is deleted (requires the container be empty) +a new container may be created with the same name without any restriction on +storage policy enforced by the deleted container which previously shared the +same name. Containers have a many-to-one relationship with policies meaning that any number -of containers can share one policy. There is no limit to how many containers can use -a specific policy. - -The notion of associating a ring with a container introduces an interesting scenario: -What would happen if 2 containers of the same name were created with different -Storage Policies on either side of a network outage at the same time? Furthermore, -what would happen if objects were placed in those containers, a whole bunch of them, -and then later the network outage was restored? Well, without special care it would -be a big problem as an application could end up using the wrong ring to try and find -an object. Luckily there is a solution for this problem, a daemon known as the -Container Reconciler works tirelessly to identify and rectify this potential scenario. +of containers can share one policy. There is no limit to how many containers +can use a specific policy. + +The notion of associating a ring with a container introduces an interesting +scenario: What would happen if 2 containers of the same name were created with +different Storage Policies on either side of a network outage at the same time? +Furthermore, what would happen if objects were placed in those containers, a +whole bunch of them, and then later the network outage was restored? Well, +without special care it would be a big problem as an application could end up +using the wrong ring to try and find an object. Luckily there is a solution for +this problem, a daemon known as the Container Reconciler works tirelessly to +identify and rectify this potential scenario. -------------------- Container Reconciler @@ -184,9 +188,9 @@ this case we would not use the default as it might not have the same policy as legacy containers. When no other policies are defined, Swift will always choose ``Policy-0`` as the default. -In other words, default means "create using this policy if nothing else is specified" -and ``Policy-0`` means "use the legacy policy if a container doesn't have one" which -really means use ``object.ring.gz`` for lookups. +In other words, default means "create using this policy if nothing else is +specified" and ``Policy-0`` means "use the legacy policy if a container doesn't +have one" which really means use ``object.ring.gz`` for lookups. .. note:: @@ -244,17 +248,19 @@ not mark the policy as deprecated to all nodes. Configuring Policies -------------------- -Policies are configured in ``swift.conf`` and it is important that the deployer have a solid -understanding of the semantics for configuring policies. Recall that a policy must have -a corresponding ring file, so configuring a policy is a two-step process. First, edit -your ``/etc/swift/swift.conf`` file to add your new policy and, second, create the -corresponding policy object ring file. +Policies are configured in ``swift.conf`` and it is important that the deployer +have a solid understanding of the semantics for configuring policies. Recall +that a policy must have a corresponding ring file, so configuring a policy is a +two-step process. First, edit your ``/etc/swift/swift.conf`` file to add your +new policy and, second, create the corresponding policy object ring file. -See :doc:`policies_saio` for a step by step guide on adding a policy to the SAIO setup. +See :doc:`policies_saio` for a step by step guide on adding a policy to the SAIO +setup. -Note that each policy has a section starting with ``[storage-policy:N]`` where N is the -policy index. There's no reason other than readability that these be sequential but there -are a number of rules enforced by Swift when parsing this file: +Note that each policy has a section starting with ``[storage-policy:N]`` where N +is the policy index. There's no reason other than readability that these be +sequential but there are a number of rules enforced by Swift when parsing this +file: * If a policy with index 0 is not declared and no other policies defined, Swift will create one @@ -269,9 +275,11 @@ are a number of rules enforced by Swift when parsing this file: * The policy name 'Policy-0' can only be used for the policy with index 0 * If any policies are defined, exactly one policy must be declared default * Deprecated policies cannot be declared the default + * If no ``policy_type`` is provided, ``replication`` is the default value. -The following is an example of a properly configured ``swift.conf`` file. See :doc:`policies_saio` -for full instructions on setting up an all-in-one with this example configuration.:: +The following is an example of a properly configured ``swift.conf`` file. See +:doc:`policies_saio` for full instructions on setting up an all-in-one with this +example configuration.:: [swift-hash] # random unique strings that can never change (DO NOT LOSE) @@ -280,10 +288,12 @@ for full instructions on setting up an all-in-one with this example configuratio [storage-policy:0] name = gold + policy_type = replication default = yes [storage-policy:1] name = silver + policy_type = replication deprecated = yes Review :ref:`default-policy` and :ref:`deprecate-policy` for more @@ -300,11 +310,14 @@ There are some other considerations when managing policies: the desired policy section, but a deprecated policy may not also be declared the default, and you must specify a default - so you must have policy which is not deprecated at all times. - -There will be additional parameters for policies as new features are added -(e.g., Erasure Code), but for now only a section name/index and name are -required. Once ``swift.conf`` is configured for a new policy, a new ring must be -created. The ring tools are not policy name aware so it's critical that the + * The option ``policy_type`` is used to distinguish between different + policy types. The default value is ``replication``. When defining an EC + policy use the value ``erasure_coding``. + * The EC policy has additional required parameters. See + :doc:`overview_erasure_code` for details. + +Once ``swift.conf`` is configured for a new policy, a new ring must be created. +The ring tools are not policy name aware so it's critical that the correct policy index be used when creating the new policy's ring file. Additional object rings are created in the same manner as the legacy ring except that '-N' is appended after the word ``object`` where N matches the @@ -404,43 +417,47 @@ Middleware ---------- Middleware can take advantage of policies through the :data:`.POLICIES` global -and by importing :func:`.get_container_info` to gain access to the policy -index associated with the container in question. From the index it -can then use the :data:`.POLICIES` singleton to grab the right ring. For example, +and by importing :func:`.get_container_info` to gain access to the policy index +associated with the container in question. From the index it can then use the +:data:`.POLICIES` singleton to grab the right ring. For example, :ref:`list_endpoints` is policy aware using the means just described. Another example is :ref:`recon` which will report the md5 sums for all of the rings. Proxy Server ------------ -The :ref:`proxy-server` module's role in Storage Policies is essentially to make sure the -correct ring is used as its member element. Before policies, the one object ring -would be instantiated when the :class:`.Application` class was instantiated and could -be overridden by test code via init parameter. With policies, however, there is -no init parameter and the :class:`.Application` class instead depends on the :data:`.POLICIES` -global singleton to retrieve the ring which is instantiated the first time it's -needed. So, instead of an object ring member of the :class:`.Application` class, there is -an accessor function, :meth:`~.Application.get_object_ring`, that gets the ring from :data:`.POLICIES`. +The :ref:`proxy-server` module's role in Storage Policies is essentially to make +sure the correct ring is used as its member element. Before policies, the one +object ring would be instantiated when the :class:`.Application` class was +instantiated and could be overridden by test code via init parameter. With +policies, however, there is no init parameter and the :class:`.Application` +class instead depends on the :data:`.POLICIES` global singleton to retrieve the +ring which is instantiated the first time it's needed. So, instead of an object +ring member of the :class:`.Application` class, there is an accessor function, +:meth:`~.Application.get_object_ring`, that gets the ring from +:data:`.POLICIES`. In general, when any module running on the proxy requires an object ring, it does so via first getting the policy index from the cached container info. The exception is during container creation where it uses the policy name from the -request header to look up policy index from the :data:`.POLICIES` global. Once the -proxy has determined the policy index, it can use the :meth:`~.Application.get_object_ring` method -described earlier to gain access to the correct ring. It then has the responsibility -of passing the index information, not the policy name, on to the back-end servers -via the header ``X-Backend-Storage-Policy-Index``. Going the other way, the proxy also -strips the index out of headers that go back to clients, and makes sure they only -see the friendly policy names. +request header to look up policy index from the :data:`.POLICIES` global. Once +the proxy has determined the policy index, it can use the +:meth:`~.Application.get_object_ring` method described earlier to gain access to +the correct ring. It then has the responsibility of passing the index +information, not the policy name, on to the back-end servers via the header ``X +-Backend-Storage-Policy-Index``. Going the other way, the proxy also strips the +index out of headers that go back to clients, and makes sure they only see the +friendly policy names. On Disk Storage --------------- -Policies each have their own directories on the back-end servers and are identified by -their storage policy indexes. Organizing the back-end directory structures by policy -index helps keep track of things and also allows for sharing of disks between policies -which may or may not make sense depending on the needs of the provider. More -on this later, but for now be aware of the following directory naming convention: +Policies each have their own directories on the back-end servers and are +identified by their storage policy indexes. Organizing the back-end directory +structures by policy index helps keep track of things and also allows for +sharing of disks between policies which may or may not make sense depending on +the needs of the provider. More on this later, but for now be aware of the +following directory naming convention: * ``/objects`` maps to objects associated with Policy-0 * ``/objects-N`` maps to storage policy index #N @@ -466,19 +483,19 @@ policy index and leaves the actual directory naming/structure mechanisms to :class:`.Diskfile` being used will assure that data is properly located in the tree based on its policy. -For the same reason, the :ref:`object-updater` also is policy aware. As previously -described, different policies use different async pending directories so the -updater needs to know how to scan them appropriately. +For the same reason, the :ref:`object-updater` also is policy aware. As +previously described, different policies use different async pending directories +so the updater needs to know how to scan them appropriately. -The :ref:`object-replicator` is policy aware in that, depending on the policy, it may have to -do drastically different things, or maybe not. For example, the difference in -handling a replication job for 2x versus 3x is trivial; however, the difference in -handling replication between 3x and erasure code is most definitely not. In -fact, the term 'replication' really isn't appropriate for some policies -like erasure code; however, the majority of the framework for collecting and -processing jobs is common. Thus, those functions in the replicator are -leveraged for all policies and then there is policy specific code required for -each policy, added when the policy is defined if needed. +The :ref:`object-replicator` is policy aware in that, depending on the policy, +it may have to do drastically different things, or maybe not. For example, the +difference in handling a replication job for 2x versus 3x is trivial; however, +the difference in handling replication between 3x and erasure code is most +definitely not. In fact, the term 'replication' really isn't appropriate for +some policies like erasure code; however, the majority of the framework for +collecting and processing jobs is common. Thus, those functions in the +replicator are leveraged for all policies and then there is policy specific code +required for each policy, added when the policy is defined if needed. The ssync functionality is policy aware for the same reason. Some of the other modules may not obviously be affected, but the back-end directory @@ -487,25 +504,26 @@ parameter. Therefore ssync being policy aware really means passing the policy index along. See :class:`~swift.obj.ssync_sender` and :class:`~swift.obj.ssync_receiver` for more information on ssync. -For :class:`.Diskfile` itself, being policy aware is all about managing the back-end -structure using the provided policy index. In other words, callers who get -a :class:`.Diskfile` instance provide a policy index and :class:`.Diskfile`'s job is to keep data -separated via this index (however it chooses) such that policies can share -the same media/nodes if desired. The included implementation of :class:`.Diskfile` -lays out the directory structure described earlier but that's owned within -:class:`.Diskfile`; external modules have no visibility into that detail. A common -function is provided to map various directory names and/or strings -based on their policy index. For example :class:`.Diskfile` defines :func:`.get_data_dir` -which builds off of a generic :func:`.get_policy_string` to consistently build -policy aware strings for various usage. +For :class:`.Diskfile` itself, being policy aware is all about managing the +back-end structure using the provided policy index. In other words, callers who +get a :class:`.Diskfile` instance provide a policy index and +:class:`.Diskfile`'s job is to keep data separated via this index (however it +chooses) such that policies can share the same media/nodes if desired. The +included implementation of :class:`.Diskfile` lays out the directory structure +described earlier but that's owned within :class:`.Diskfile`; external modules +have no visibility into that detail. A common function is provided to map +various directory names and/or strings based on their policy index. For example +:class:`.Diskfile` defines :func:`.get_data_dir` which builds off of a generic +:func:`.get_policy_string` to consistently build policy aware strings for +various usage. Container Server ---------------- -The :ref:`container-server` plays a very important role in Storage Policies, it is -responsible for handling the assignment of a policy to a container and the -prevention of bad things like changing policies or picking the wrong policy -to use when nothing is specified (recall earlier discussion on Policy-0 versus +The :ref:`container-server` plays a very important role in Storage Policies, it +is responsible for handling the assignment of a policy to a container and the +prevention of bad things like changing policies or picking the wrong policy to +use when nothing is specified (recall earlier discussion on Policy-0 versus default). The :ref:`container-updater` is policy aware, however its job is very simple, to @@ -538,19 +556,19 @@ migrated to be fully compatible with the post-storage-policy queries without having to fall back and retry queries with the legacy schema to service container read requests. -The :ref:`container-sync-daemon` functionality only needs to be policy aware in that it -accesses the object rings. Therefore, it needs to pull the policy index -out of the container information and use it to select the appropriate -object ring from the :data:`.POLICIES` global. +The :ref:`container-sync-daemon` functionality only needs to be policy aware in +that it accesses the object rings. Therefore, it needs to pull the policy index +out of the container information and use it to select the appropriate object +ring from the :data:`.POLICIES` global. Account Server -------------- -The :ref:`account-server`'s role in Storage Policies is really limited to reporting. -When a HEAD request is made on an account (see example provided earlier), -the account server is provided with the storage policy index and builds -the ``object_count`` and ``byte_count`` information for the client on a per -policy basis. +The :ref:`account-server`'s role in Storage Policies is really limited to +reporting. When a HEAD request is made on an account (see example provided +earlier), the account server is provided with the storage policy index and +builds the ``object_count`` and ``byte_count`` information for the client on a +per policy basis. The account servers are able to report per-storage-policy object and byte counts because of some policy specific DB schema changes. A policy specific @@ -564,23 +582,23 @@ pre-storage-policy accounts by altering the DB schema and populating the point in time. The per-storage-policy object and byte counts are not updated with each object -PUT and DELETE request, instead container updates to the account server are performed -asynchronously by the ``swift-container-updater``. +PUT and DELETE request, instead container updates to the account server are +performed asynchronously by the ``swift-container-updater``. .. _upgrade-policy: Upgrading and Confirming Functionality -------------------------------------- -Upgrading to a version of Swift that has Storage Policy support is not difficult, -in fact, the cluster administrator isn't required to make any special configuration -changes to get going. Swift will automatically begin using the existing object -ring as both the default ring and the Policy-0 ring. Adding the declaration of -policy 0 is totally optional and in its absence, the name given to the implicit -policy 0 will be 'Policy-0'. Let's say for testing purposes that you wanted to take -an existing cluster that already has lots of data on it and upgrade to Swift with -Storage Policies. From there you want to go ahead and create a policy and test a -few things out. All you need to do is: +Upgrading to a version of Swift that has Storage Policy support is not +difficult, in fact, the cluster administrator isn't required to make any special +configuration changes to get going. Swift will automatically begin using the +existing object ring as both the default ring and the Policy-0 ring. Adding the +declaration of policy 0 is totally optional and in its absence, the name given +to the implicit policy 0 will be 'Policy-0'. Let's say for testing purposes +that you wanted to take an existing cluster that already has lots of data on it +and upgrade to Swift with Storage Policies. From there you want to go ahead and +create a policy and test a few things out. All you need to do is: #. Upgrade all of your Swift nodes to a policy-aware version of Swift #. Define your policies in ``/etc/swift/swift.conf`` diff --git a/doc/source/overview_replication.rst b/doc/source/overview_replication.rst index 81523fab5..56aeeacd7 100644 --- a/doc/source/overview_replication.rst +++ b/doc/source/overview_replication.rst @@ -111,11 +111,53 @@ Another improvement planned all along the way is separating the local disk structure from the protocol path structure. This separation will allow ring resizing at some point, or at least ring-doubling. -FOR NOW, IT IS NOT RECOMMENDED TO USE SSYNC ON PRODUCTION CLUSTERS. Some of us -will be in a limited fashion to look for any subtle issues, tuning, etc. but -generally ssync is an experimental feature. In its current implementation it is -probably going to be a bit slower than RSync, but if all goes according to plan -it will end up much faster. +Note that for objects being stored with an Erasure Code policy, the replicator +daemon is not involved. Instead, the reconstructor is used by Erasure Code +policies and is analogous to the replicator for Replication type policies. +See :doc:`overview_erasure_code` for complete information on both Erasure Code +support as well as the reconstructor. + +---------- +Hashes.pkl +---------- + +The hashes.pkl file is a key element for both replication and reconstruction +(for Erasure Coding). Both daemons use this file to determine if any kind of +action is required between nodes that are participating in the durability +scheme. The file itself is a pickled dictionary with slightly different +formats depending on whether the policy is Replication or Erasure Code. In +either case, however, the same basic information is provided between the +nodes. The dictionary contains a dictionary where the key is a suffix +directory name and the value is the MD5 hash of the directory listing for +that suffix. In this manner, the daemon can quickly identify differences +between local and remote suffix directories on a per partition basis as the +scope of any one hashes.pkl file is a partition directory. + +For Erasure Code policies, there is a little more information required. An +object's hash directory may contain multiple fragments of a single object in +the event that the node is acting as a handoff or perhaps if a rebalance is +underway. Each fragment of an object is stored with a fragment index, so +the hashes.pkl for an Erasure Code partition will still be a dictionary +keyed on the suffix directory name, however, the value is another dictionary +keyed on the fragment index with subsequent MD5 hashes for each one as +values. Some files within an object hash directory don't require a fragment +index so None is used to represent those. Below are examples of what these +dictionaries might look like. + +Replication hashes.pkl:: + + {'a43': '72018c5fbfae934e1f56069ad4425627', + 'b23': '12348c5fbfae934e1f56069ad4421234'} + +Erasure Code hashes.pkl:: + + {'a43': {None: '72018c5fbfae934e1f56069ad4425627', + 2: 'b6dd6db937cb8748f50a5b6e4bc3b808'}, + 'b23': {None: '12348c5fbfae934e1f56069ad4421234', + 1: '45676db937cb8748f50a5b6e4bc34567'}} + + + ----------------------------- diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 6e881d9e0..e7b8a802f 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -170,6 +170,11 @@ use = egg:swift#recon # # Maximum amount of time in seconds for the connection attempt # conn_timeout = 5 +# Server errors from requests will be retried by default +# request_tries = 3 +# +# Internal client config file path +# internal_client_conf_path = /etc/swift/internal-client.conf # Note: Put it at the beginning of the pipeline to profile all middleware. But # it is safer to put this after healthcheck. diff --git a/etc/internal-client.conf-sample b/etc/internal-client.conf-sample new file mode 100644 index 000000000..2d25d448b --- /dev/null +++ b/etc/internal-client.conf-sample @@ -0,0 +1,42 @@ +[DEFAULT] +# swift_dir = /etc/swift +# user = swift +# You can specify default log routing here if you want: +# log_name = swift +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# +# comma separated list of functions to call to setup custom log handlers. +# functions get passed: conf, name, log_to_console, log_route, fmt, logger, +# adapted_logger +# log_custom_handlers = +# +# If set, log_udp_host will override log_address +# log_udp_host = +# log_udp_port = 514 +# +# You can enable StatsD logging here: +# log_statsd_host = localhost +# log_statsd_port = 8125 +# log_statsd_default_sample_rate = 1.0 +# log_statsd_sample_rate_factor = 1.0 +# log_statsd_metric_prefix = + +[pipeline:main] +pipeline = catch_errors proxy-logging cache proxy-server + +[app:proxy-server] +use = egg:swift#proxy +# See proxy-server.conf-sample for options + +[filter:cache] +use = egg:swift#memcache +# See proxy-server.conf-sample for options + +[filter:proxy-logging] +use = egg:swift#proxy_logging + +[filter:catch_errors] +use = egg:swift#catch_errors +# See proxy-server.conf-sample for options diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index b594a9576..c510e0fb2 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -211,6 +211,29 @@ use = egg:swift#recon # removed when it has successfully replicated to all the canonical nodes. # handoff_delete = auto +[object-reconstructor] +# You can override the default log routing for this app here (don't use set!): +# Unless otherwise noted, each setting below has the same meaning as described +# in the [object-replicator] section, however these settings apply to the EC +# reconstructor +# +# log_name = object-reconstructor +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# +# daemonize = on +# run_pause = 30 +# concurrency = 1 +# stats_interval = 300 +# node_timeout = 10 +# http_timeout = 60 +# lockup_timeout = 1800 +# reclaim_age = 604800 +# ring_check_interval = 15 +# recon_cache_path = /var/cache/swift +# handoffs_first = False + [object-updater] # You can override the default log routing for this app here (don't use set!): # log_name = object-updater diff --git a/etc/swift.conf-sample b/etc/swift.conf-sample index fac17676c..872681401 100644 --- a/etc/swift.conf-sample +++ b/etc/swift.conf-sample @@ -22,9 +22,13 @@ swift_hash_path_prefix = changeme # defined you must define a policy with index 0 and you must specify a # default. It is recommended you always define a section for # storage-policy:0. +# +# A 'policy_type' argument is also supported but is not mandatory. Default +# policy type 'replication' is used when 'policy_type' is unspecified. [storage-policy:0] name = Policy-0 default = yes +#policy_type = replication # the following section would declare a policy called 'silver', the number of # replicas will be determined by how the ring is built. In this example the @@ -39,9 +43,45 @@ default = yes # current default. #[storage-policy:1] #name = silver +#policy_type = replication + +# The following declares a storage policy of type 'erasure_coding' which uses +# Erasure Coding for data reliability. The 'erasure_coding' storage policy in +# Swift is available as a "beta". Please refer to Swift documentation for +# details on how the 'erasure_coding' storage policy is implemented. +# +# Swift uses PyECLib, a Python Erasure coding API library, for encode/decode +# operations. Please refer to Swift documentation for details on how to +# install PyECLib. +# +# When defining an EC policy, 'policy_type' needs to be 'erasure_coding' and +# EC configuration parameters 'ec_type', 'ec_num_data_fragments' and +# 'ec_num_parity_fragments' must be specified. 'ec_type' is chosen from the +# list of EC backends supported by PyECLib. The ring configured for the +# storage policy must have it's "replica" count configured to +# 'ec_num_data_fragments' + 'ec_num_parity_fragments' - this requirement is +# validated when services start. 'ec_object_segment_size' is the amount of +# data that will be buffered up before feeding a segment into the +# encoder/decoder. More information about these configuration options and +# supported `ec_type` schemes is available in the Swift documentation. Please +# refer to Swift documentation for details on how to configure EC policies. +# +# The example 'deepfreeze10-4' policy defined below is a _sample_ +# configuration with 10 'data' and 4 'parity' fragments. 'ec_type' +# defines the Erasure Coding scheme. 'jerasure_rs_vand' (Reed-Solomon +# Vandermonde) is used as an example below. +# +#[storage-policy:2] +#name = deepfreeze10-4 +#policy_type = erasure_coding +#ec_type = jerasure_rs_vand +#ec_num_data_fragments = 10 +#ec_num_parity_fragments = 4 +#ec_object_segment_size = 1048576 + # The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. These constraints are automatically +# saved in the swift cluster. These constraints are automatically # published by the proxy server in responses to /info requests. [swift-constraints] @@ -51,6 +51,7 @@ scripts = bin/swift-object-expirer bin/swift-object-info bin/swift-object-replicator + bin/swift-object-reconstructor bin/swift-object-server bin/swift-object-updater bin/swift-oldies diff --git a/swift/account/reaper.py b/swift/account/reaper.py index ce69fab92..06a008535 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -19,6 +19,7 @@ from swift import gettext_ as _ from logging import DEBUG from math import sqrt from time import time +import itertools from eventlet import GreenPool, sleep, Timeout @@ -432,7 +433,7 @@ class AccountReaper(Daemon): * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the container node dicts. """ - container_nodes = list(container_nodes) + cnodes = itertools.cycle(container_nodes) try: ring = self.get_object_ring(policy_index) except PolicyError: @@ -443,7 +444,7 @@ class AccountReaper(Daemon): successes = 0 failures = 0 for node in nodes: - cnode = container_nodes.pop() + cnode = next(cnodes) try: direct_delete_object( node, part, account, container, obj, diff --git a/swift/cli/info.py b/swift/cli/info.py index 142b103f4..a8cfabd17 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -24,7 +24,7 @@ from swift.common.request_helpers import is_sys_meta, is_user_meta, \ from swift.account.backend import AccountBroker, DATADIR as ABDATADIR from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \ - extract_policy_index + extract_policy from swift.common.storage_policy import POLICIES @@ -341,10 +341,7 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', datadir = DATADIR_BASE # try to extract policy index from datafile disk path - try: - policy_index = extract_policy_index(datafile) - except ValueError: - pass + policy_index = int(extract_policy(datafile) or POLICIES.legacy) try: if policy_index: diff --git a/swift/common/constraints.py b/swift/common/constraints.py index d4458ddf8..8e3ba53b0 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -204,6 +204,19 @@ def check_object_creation(req, object_name): return check_metadata(req, 'object') +def check_dir(root, drive): + """ + Verify that the path to the device is a directory and is a lesser + constraint that is enforced when a full mount_check isn't possible + with, for instance, a VM using loopback or partitions. + + :param root: base path where the dir is + :param drive: drive name to be checked + :returns: True if it is a valid directoy, False otherwise + """ + return os.path.isdir(os.path.join(root, drive)) + + def check_mount(root, drive): """ Verify that the path to the device is a mount point and mounted. This diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index d7ea759d6..b4c926eb1 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -31,10 +31,32 @@ class SwiftException(Exception): pass +class PutterConnectError(Exception): + + def __init__(self, status=None): + self.status = status + + class InvalidTimestamp(SwiftException): pass +class InsufficientStorage(SwiftException): + pass + + +class FooterNotSupported(SwiftException): + pass + + +class MultiphasePUTNotSupported(SwiftException): + pass + + +class SuffixSyncError(SwiftException): + pass + + class DiskFileError(SwiftException): pass @@ -103,6 +125,10 @@ class ConnectionTimeout(Timeout): pass +class ResponseTimeout(Timeout): + pass + + class DriveNotMounted(SwiftException): pass diff --git a/swift/common/manager.py b/swift/common/manager.py index a4ef350da..260516d6a 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -33,7 +33,8 @@ ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', 'container-replicator', 'container-reconciler', 'container-server', 'container-sync', 'container-updater', 'object-auditor', 'object-server', - 'object-expirer', 'object-replicator', 'object-updater', + 'object-expirer', 'object-replicator', + 'object-reconstructor', 'object-updater', 'proxy-server', 'account-replicator', 'account-reaper'] MAIN_SERVERS = ['proxy-server', 'account-server', 'container-server', 'object-server'] diff --git a/swift/common/middleware/formpost.py b/swift/common/middleware/formpost.py index 7132b342a..56a6d20f3 100644 --- a/swift/common/middleware/formpost.py +++ b/swift/common/middleware/formpost.py @@ -218,7 +218,14 @@ class FormPost(object): env, attrs['boundary']) start_response(status, headers) return [body] - except (FormInvalid, MimeInvalid, EOFError) as err: + except MimeInvalid: + body = 'FormPost: invalid starting boundary' + start_response( + '400 Bad Request', + (('Content-Type', 'text/plain'), + ('Content-Length', str(len(body))))) + return [body] + except (FormInvalid, EOFError) as err: body = 'FormPost: %s' % err start_response( '400 Bad Request', diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index 08e0ab5dc..14b9fd884 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -26,10 +26,12 @@ import time from contextlib import contextmanager from urllib import unquote from swift import gettext_ as _ +from swift.common.storage_policy import POLICIES from swift.common.constraints import FORMAT2CONTENT_TYPE from swift.common.exceptions import ListingIterError, SegmentError from swift.common.http import is_success -from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable +from swift.common.swob import (HTTPBadRequest, HTTPNotAcceptable, + HTTPServiceUnavailable) from swift.common.utils import split_path, validate_device_partition from swift.common.wsgi import make_subrequest @@ -82,21 +84,27 @@ def get_listing_content_type(req): def get_name_and_placement(request, minsegs=1, maxsegs=None, rest_with_last=False): """ - Utility function to split and validate the request path and - storage_policy_index. The storage_policy_index is extracted from - the headers of the request and converted to an integer, and then the - args are passed through to :meth:`split_and_validate_path`. + Utility function to split and validate the request path and storage + policy. The storage policy index is extracted from the headers of + the request and converted to a StoragePolicy instance. The + remaining args are passed through to + :meth:`split_and_validate_path`. :returns: a list, result of :meth:`split_and_validate_path` with - storage_policy_index appended on the end - :raises: HTTPBadRequest + the BaseStoragePolicy instance appended on the end + :raises: HTTPServiceUnavailable if the path is invalid or no policy exists + with the extracted policy_index. """ - policy_idx = request.headers.get('X-Backend-Storage-Policy-Index', '0') - policy_idx = int(policy_idx) + policy_index = request.headers.get('X-Backend-Storage-Policy-Index') + policy = POLICIES.get_by_index(policy_index) + if not policy: + raise HTTPServiceUnavailable( + body=_("No policy with index %s") % policy_index, + request=request, content_type='text/plain') results = split_and_validate_path(request, minsegs=minsegs, maxsegs=maxsegs, rest_with_last=rest_with_last) - results.append(policy_idx) + results.append(policy) return results diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index daad23ff1..62e19951d 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -243,7 +243,7 @@ class Ring(object): if dev_id not in seen_ids: part_nodes.append(self.devs[dev_id]) seen_ids.add(dev_id) - return part_nodes + return [dict(node, index=i) for i, node in enumerate(part_nodes)] def get_part(self, account, container=None, obj=None): """ @@ -291,6 +291,7 @@ class Ring(object): ====== =============================================================== id unique integer identifier amongst devices + index offset into the primary node list for the partition weight a float of the relative weight of this device as compared to others; this indicates how many partitions the builder will try to assign to this device diff --git a/swift/common/storage_policy.py b/swift/common/storage_policy.py index f33eda539..e45ab018c 100644 --- a/swift/common/storage_policy.py +++ b/swift/common/storage_policy.py @@ -17,10 +17,18 @@ import string from swift.common.utils import config_true_value, SWIFT_CONF_FILE from swift.common.ring import Ring +from swift.common.utils import quorum_size +from swift.common.exceptions import RingValidationError +from pyeclib.ec_iface import ECDriver, ECDriverError, VALID_EC_TYPES LEGACY_POLICY_NAME = 'Policy-0' VALID_CHARS = '-' + string.letters + string.digits +DEFAULT_POLICY_TYPE = REPL_POLICY = 'replication' +EC_POLICY = 'erasure_coding' + +DEFAULT_EC_OBJECT_SEGMENT_SIZE = 1048576 + class PolicyError(ValueError): @@ -38,36 +46,73 @@ def _get_policy_string(base, policy_index): return return_string -def get_policy_string(base, policy_index): +def get_policy_string(base, policy_or_index): """ - Helper function to construct a string from a base and the policy - index. Used to encode the policy index into either a file name - or a directory name by various modules. + Helper function to construct a string from a base and the policy. + Used to encode the policy index into either a file name or a + directory name by various modules. :param base: the base string - :param policy_index: the storage policy index + :param policy_or_index: StoragePolicy instance, or an index + (string or int), if None the legacy + storage Policy-0 is assumed. :returns: base name with policy index added + :raises: PolicyError if no policy exists with the given policy_index """ - if POLICIES.get_by_index(policy_index) is None: - raise PolicyError("No policy with index %r" % policy_index) - return _get_policy_string(base, policy_index) + if isinstance(policy_or_index, BaseStoragePolicy): + policy = policy_or_index + else: + policy = POLICIES.get_by_index(policy_or_index) + if policy is None: + raise PolicyError("Unknown policy", index=policy_or_index) + return _get_policy_string(base, int(policy)) -class StoragePolicy(object): +def split_policy_string(policy_string): """ - Represents a storage policy. - Not meant to be instantiated directly; use - :func:`~swift.common.storage_policy.reload_storage_policies` to load - POLICIES from ``swift.conf``. + Helper function to convert a string representing a base and a + policy. Used to decode the policy from either a file name or + a directory name by various modules. + + :param policy_string: base name with policy index added + + :raises: PolicyError if given index does not map to a valid policy + :returns: a tuple, in the form (base, policy) where base is the base + string and policy is the StoragePolicy instance for the + index encoded in the policy_string. + """ + if '-' in policy_string: + base, policy_index = policy_string.rsplit('-', 1) + else: + base, policy_index = policy_string, None + policy = POLICIES.get_by_index(policy_index) + if get_policy_string(base, policy) != policy_string: + raise PolicyError("Unknown policy", index=policy_index) + return base, policy + + +class BaseStoragePolicy(object): + """ + Represents a storage policy. Not meant to be instantiated directly; + implement a derived subclasses (e.g. StoragePolicy, ECStoragePolicy, etc) + or use :func:`~swift.common.storage_policy.reload_storage_policies` to + load POLICIES from ``swift.conf``. The object_ring property is lazy loaded once the service's ``swift_dir`` is known via :meth:`~StoragePolicyCollection.get_object_ring`, but it may be over-ridden via object_ring kwarg at create time for testing or actively loaded with :meth:`~StoragePolicy.load_ring`. """ + + policy_type_to_policy_cls = {} + def __init__(self, idx, name='', is_default=False, is_deprecated=False, object_ring=None): + # do not allow BaseStoragePolicy class to be instantiated directly + if type(self) == BaseStoragePolicy: + raise TypeError("Can't instantiate BaseStoragePolicy directly") + # policy parameter validation try: self.idx = int(idx) except ValueError: @@ -88,6 +133,8 @@ class StoragePolicy(object): self.name = name self.is_deprecated = config_true_value(is_deprecated) self.is_default = config_true_value(is_default) + if self.policy_type not in BaseStoragePolicy.policy_type_to_policy_cls: + raise PolicyError('Invalid type', self.policy_type) if self.is_deprecated and self.is_default: raise PolicyError('Deprecated policy can not be default. ' 'Invalid config', self.idx) @@ -101,8 +148,80 @@ class StoragePolicy(object): return cmp(self.idx, int(other)) def __repr__(self): - return ("StoragePolicy(%d, %r, is_default=%s, is_deprecated=%s)") % ( - self.idx, self.name, self.is_default, self.is_deprecated) + return ("%s(%d, %r, is_default=%s, " + "is_deprecated=%s, policy_type=%r)") % \ + (self.__class__.__name__, self.idx, self.name, + self.is_default, self.is_deprecated, self.policy_type) + + @classmethod + def register(cls, policy_type): + """ + Decorator for Storage Policy implementations to register + their StoragePolicy class. This will also set the policy_type + attribute on the registered implementation. + """ + def register_wrapper(policy_cls): + if policy_type in cls.policy_type_to_policy_cls: + raise PolicyError( + '%r is already registered for the policy_type %r' % ( + cls.policy_type_to_policy_cls[policy_type], + policy_type)) + cls.policy_type_to_policy_cls[policy_type] = policy_cls + policy_cls.policy_type = policy_type + return policy_cls + return register_wrapper + + @classmethod + def _config_options_map(cls): + """ + Map config option name to StoragePolicy parameter name. + """ + return { + 'name': 'name', + 'policy_type': 'policy_type', + 'default': 'is_default', + 'deprecated': 'is_deprecated', + } + + @classmethod + def from_config(cls, policy_index, options): + config_to_policy_option_map = cls._config_options_map() + policy_options = {} + for config_option, value in options.items(): + try: + policy_option = config_to_policy_option_map[config_option] + except KeyError: + raise PolicyError('Invalid option %r in ' + 'storage-policy section' % config_option, + index=policy_index) + policy_options[policy_option] = value + return cls(policy_index, **policy_options) + + def get_info(self, config=False): + """ + Return the info dict and conf file options for this policy. + + :param config: boolean, if True all config options are returned + """ + info = {} + for config_option, policy_attribute in \ + self._config_options_map().items(): + info[config_option] = getattr(self, policy_attribute) + if not config: + # remove some options for public consumption + if not self.is_default: + info.pop('default') + if not self.is_deprecated: + info.pop('deprecated') + info.pop('policy_type') + return info + + def _validate_ring(self): + """ + Hook, called when the ring is loaded. Can be used to + validate the ring against the StoragePolicy configuration. + """ + pass def load_ring(self, swift_dir): """ @@ -114,11 +233,224 @@ class StoragePolicy(object): return self.object_ring = Ring(swift_dir, ring_name=self.ring_name) - def get_options(self): - """Return the valid conf file options for this policy.""" - return {'name': self.name, - 'default': self.is_default, - 'deprecated': self.is_deprecated} + # Validate ring to make sure it conforms to policy requirements + self._validate_ring() + + @property + def quorum(self): + """ + Number of successful backend requests needed for the proxy to + consider the client request successful. + """ + raise NotImplementedError() + + +@BaseStoragePolicy.register(REPL_POLICY) +class StoragePolicy(BaseStoragePolicy): + """ + Represents a storage policy of type 'replication'. Default storage policy + class unless otherwise overridden from swift.conf. + + Not meant to be instantiated directly; use + :func:`~swift.common.storage_policy.reload_storage_policies` to load + POLICIES from ``swift.conf``. + """ + + @property + def quorum(self): + """ + Quorum concept in the replication case: + floor(number of replica / 2) + 1 + """ + if not self.object_ring: + raise PolicyError('Ring is not loaded') + return quorum_size(self.object_ring.replica_count) + + +@BaseStoragePolicy.register(EC_POLICY) +class ECStoragePolicy(BaseStoragePolicy): + """ + Represents a storage policy of type 'erasure_coding'. + + Not meant to be instantiated directly; use + :func:`~swift.common.storage_policy.reload_storage_policies` to load + POLICIES from ``swift.conf``. + """ + def __init__(self, idx, name='', is_default=False, + is_deprecated=False, object_ring=None, + ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, + ec_type=None, ec_ndata=None, ec_nparity=None): + + super(ECStoragePolicy, self).__init__( + idx, name, is_default, is_deprecated, object_ring) + + # Validate erasure_coding policy specific members + # ec_type is one of the EC implementations supported by PyEClib + if ec_type is None: + raise PolicyError('Missing ec_type') + if ec_type not in VALID_EC_TYPES: + raise PolicyError('Wrong ec_type %s for policy %s, should be one' + ' of "%s"' % (ec_type, self.name, + ', '.join(VALID_EC_TYPES))) + self._ec_type = ec_type + + # Define _ec_ndata as the number of EC data fragments + # Accessible as the property "ec_ndata" + try: + value = int(ec_ndata) + if value <= 0: + raise ValueError + self._ec_ndata = value + except (TypeError, ValueError): + raise PolicyError('Invalid ec_num_data_fragments %r' % + ec_ndata, index=self.idx) + + # Define _ec_nparity as the number of EC parity fragments + # Accessible as the property "ec_nparity" + try: + value = int(ec_nparity) + if value <= 0: + raise ValueError + self._ec_nparity = value + except (TypeError, ValueError): + raise PolicyError('Invalid ec_num_parity_fragments %r' + % ec_nparity, index=self.idx) + + # Define _ec_segment_size as the encode segment unit size + # Accessible as the property "ec_segment_size" + try: + value = int(ec_segment_size) + if value <= 0: + raise ValueError + self._ec_segment_size = value + except (TypeError, ValueError): + raise PolicyError('Invalid ec_object_segment_size %r' % + ec_segment_size, index=self.idx) + + # Initialize PyECLib EC backend + try: + self.pyeclib_driver = \ + ECDriver(k=self._ec_ndata, m=self._ec_nparity, + ec_type=self._ec_type) + except ECDriverError as e: + raise PolicyError("Error creating EC policy (%s)" % e, + index=self.idx) + + # quorum size in the EC case depends on the choice of EC scheme. + self._ec_quorum_size = \ + self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed() + + @property + def ec_type(self): + return self._ec_type + + @property + def ec_ndata(self): + return self._ec_ndata + + @property + def ec_nparity(self): + return self._ec_nparity + + @property + def ec_segment_size(self): + return self._ec_segment_size + + @property + def fragment_size(self): + """ + Maximum length of a fragment, including header. + + NB: a fragment archive is a sequence of 0 or more max-length + fragments followed by one possibly-shorter fragment. + """ + # Technically pyeclib's get_segment_info signature calls for + # (data_len, segment_size) but on a ranged GET we don't know the + # ec-content-length header before we need to compute where in the + # object we should request to align with the fragment size. So we + # tell pyeclib a lie - from it's perspective, as long as data_len >= + # segment_size it'll give us the answer we want. From our + # perspective, because we only use this answer to calculate the + # *minimum* size we should read from an object body even if data_len < + # segment_size we'll still only read *the whole one and only last + # fragment* and pass than into pyeclib who will know what to do with + # it just as it always does when the last fragment is < fragment_size. + return self.pyeclib_driver.get_segment_info( + self.ec_segment_size, self.ec_segment_size)['fragment_size'] + + @property + def ec_scheme_description(self): + """ + This short hand form of the important parts of the ec schema is stored + in Object System Metadata on the EC Fragment Archives for debugging. + """ + return "%s %d+%d" % (self._ec_type, self._ec_ndata, self._ec_nparity) + + def __repr__(self): + return ("%s, EC config(ec_type=%s, ec_segment_size=%d, " + "ec_ndata=%d, ec_nparity=%d)") % ( + super(ECStoragePolicy, self).__repr__(), self.ec_type, + self.ec_segment_size, self.ec_ndata, self.ec_nparity) + + @classmethod + def _config_options_map(cls): + options = super(ECStoragePolicy, cls)._config_options_map() + options.update({ + 'ec_type': 'ec_type', + 'ec_object_segment_size': 'ec_segment_size', + 'ec_num_data_fragments': 'ec_ndata', + 'ec_num_parity_fragments': 'ec_nparity', + }) + return options + + def get_info(self, config=False): + info = super(ECStoragePolicy, self).get_info(config=config) + if not config: + info.pop('ec_object_segment_size') + info.pop('ec_num_data_fragments') + info.pop('ec_num_parity_fragments') + info.pop('ec_type') + return info + + def _validate_ring(self): + """ + EC specific validation + + Replica count check - we need _at_least_ (#data + #parity) replicas + configured. Also if the replica count is larger than exactly that + number there's a non-zero risk of error for code that is considering + the number of nodes in the primary list from the ring. + """ + if not self.object_ring: + raise PolicyError('Ring is not loaded') + nodes_configured = self.object_ring.replica_count + if nodes_configured != (self.ec_ndata + self.ec_nparity): + raise RingValidationError( + 'EC ring for policy %s needs to be configured with ' + 'exactly %d nodes. Got %d.' % (self.name, + self.ec_ndata + self.ec_nparity, nodes_configured)) + + @property + def quorum(self): + """ + Number of successful backend requests needed for the proxy to consider + the client request successful. + + The quorum size for EC policies defines the minimum number + of data + parity elements required to be able to guarantee + the desired fault tolerance, which is the number of data + elements supplemented by the minimum number of parity + elements required by the chosen erasure coding scheme. + + For example, for Reed-Solomon, the minimum number parity + elements required is 1, and thus the quorum_size requirement + is ec_ndata + 1. + + Given the number of parity elements required is not the same + for every erasure coding scheme, consult PyECLib for + min_parity_fragments_needed() + """ + return self._ec_quorum_size class StoragePolicyCollection(object): @@ -236,9 +568,19 @@ class StoragePolicyCollection(object): :returns: storage policy, or None if no such policy """ # makes it easier for callers to just pass in a header value - index = int(index) if index else 0 + if index in ('', None): + index = 0 + else: + try: + index = int(index) + except ValueError: + return None return self.by_index.get(index) + @property + def legacy(self): + return self.get_by_index(None) + def get_object_ring(self, policy_idx, swift_dir): """ Get the ring object to use to handle a request based on its policy. @@ -267,10 +609,7 @@ class StoragePolicyCollection(object): # delete from /info if deprecated if pol.is_deprecated: continue - policy_entry = {} - policy_entry['name'] = pol.name - if pol.is_default: - policy_entry['default'] = pol.is_default + policy_entry = pol.get_info() policy_info.append(policy_entry) return policy_info @@ -287,22 +626,10 @@ def parse_storage_policies(conf): if not section.startswith('storage-policy:'): continue policy_index = section.split(':', 1)[1] - # map config option name to StoragePolicy parameter name - config_to_policy_option_map = { - 'name': 'name', - 'default': 'is_default', - 'deprecated': 'is_deprecated', - } - policy_options = {} - for config_option, value in conf.items(section): - try: - policy_option = config_to_policy_option_map[config_option] - except KeyError: - raise PolicyError('Invalid option %r in ' - 'storage-policy section %r' % ( - config_option, section)) - policy_options[policy_option] = value - policy = StoragePolicy(policy_index, **policy_options) + config_options = dict(conf.items(section)) + policy_type = config_options.pop('policy_type', DEFAULT_POLICY_TYPE) + policy_cls = BaseStoragePolicy.policy_type_to_policy_cls[policy_type] + policy = policy_cls.from_config(policy_index, config_options) policies.append(policy) return StoragePolicyCollection(policies) diff --git a/swift/common/swob.py b/swift/common/swob.py index 729cdd96f..c2e3afb4e 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -36,7 +36,7 @@ needs to change. """ from collections import defaultdict -from cStringIO import StringIO +from StringIO import StringIO import UserDict import time from functools import partial @@ -128,6 +128,20 @@ class _UTC(tzinfo): UTC = _UTC() +class WsgiStringIO(StringIO): + """ + This class adds support for the additional wsgi.input methods defined on + eventlet.wsgi.Input to the StringIO class which would otherwise be a fine + stand-in for the file-like object in the WSGI environment. + """ + + def set_hundred_continue_response_headers(self, headers): + pass + + def send_hundred_continue_response(self): + pass + + def _datetime_property(header): """ Set and retrieve the datetime value of self.headers[header] @@ -743,16 +757,16 @@ def _req_environ_property(environ_field): def _req_body_property(): """ Set and retrieve the Request.body parameter. It consumes wsgi.input and - returns the results. On assignment, uses a StringIO to create a new + returns the results. On assignment, uses a WsgiStringIO to create a new wsgi.input. """ def getter(self): body = self.environ['wsgi.input'].read() - self.environ['wsgi.input'] = StringIO(body) + self.environ['wsgi.input'] = WsgiStringIO(body) return body def setter(self, value): - self.environ['wsgi.input'] = StringIO(value) + self.environ['wsgi.input'] = WsgiStringIO(value) self.environ['CONTENT_LENGTH'] = str(len(value)) return property(getter, setter, doc="Get and set the request body str") @@ -820,7 +834,7 @@ class Request(object): :param path: encoded, parsed, and unquoted into PATH_INFO :param environ: WSGI environ dictionary :param headers: HTTP headers - :param body: stuffed in a StringIO and hung on wsgi.input + :param body: stuffed in a WsgiStringIO and hung on wsgi.input :param kwargs: any environ key with an property setter """ headers = headers or {} @@ -855,10 +869,10 @@ class Request(object): } env.update(environ) if body is not None: - env['wsgi.input'] = StringIO(body) + env['wsgi.input'] = WsgiStringIO(body) env['CONTENT_LENGTH'] = str(len(body)) elif 'wsgi.input' not in env: - env['wsgi.input'] = StringIO('') + env['wsgi.input'] = WsgiStringIO('') req = Request(env) for key, val in headers.iteritems(): req.headers[key] = val @@ -965,7 +979,7 @@ class Request(object): env.update({ 'REQUEST_METHOD': 'GET', 'CONTENT_LENGTH': '0', - 'wsgi.input': StringIO(''), + 'wsgi.input': WsgiStringIO(''), }) return Request(env) @@ -1102,10 +1116,12 @@ class Response(object): app_iter = _resp_app_iter_property() def __init__(self, body=None, status=200, headers=None, app_iter=None, - request=None, conditional_response=False, **kw): + request=None, conditional_response=False, + conditional_etag=None, **kw): self.headers = HeaderKeyDict( [('Content-Type', 'text/html; charset=UTF-8')]) self.conditional_response = conditional_response + self._conditional_etag = conditional_etag self.request = request self.body = body self.app_iter = app_iter @@ -1131,6 +1147,26 @@ class Response(object): if 'charset' in kw and 'content_type' in kw: self.charset = kw['charset'] + @property + def conditional_etag(self): + """ + The conditional_etag keyword argument for Response will allow the + conditional match value of a If-Match request to be compared to a + non-standard value. + + This is available for Storage Policies that do not store the client + object data verbatim on the storage nodes, but still need support + conditional requests. + + It's most effectively used with X-Backend-Etag-Is-At which would + define the additional Metadata key where the original ETag of the + clear-form client request data. + """ + if self._conditional_etag is not None: + return self._conditional_etag + else: + return self.etag + def _prepare_for_ranges(self, ranges): """ Prepare the Response for multiple ranges. @@ -1161,15 +1197,16 @@ class Response(object): return content_size, content_type def _response_iter(self, app_iter, body): + etag = self.conditional_etag if self.conditional_response and self.request: - if self.etag and self.request.if_none_match and \ - self.etag in self.request.if_none_match: + if etag and self.request.if_none_match and \ + etag in self.request.if_none_match: self.status = 304 self.content_length = 0 return [''] - if self.etag and self.request.if_match and \ - self.etag not in self.request.if_match: + if etag and self.request.if_match and \ + etag not in self.request.if_match: self.status = 412 self.content_length = 0 return [''] diff --git a/swift/common/utils.py b/swift/common/utils.py index cf7b7e7c5..19dcfd3d6 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -2236,11 +2236,16 @@ class GreenAsyncPile(object): Correlating results with jobs (if necessary) is left to the caller. """ - def __init__(self, size): + def __init__(self, size_or_pool): """ - :param size: size pool of green threads to use + :param size_or_pool: thread pool size or a pool to use """ - self._pool = GreenPool(size) + if isinstance(size_or_pool, GreenPool): + self._pool = size_or_pool + size = self._pool.size + else: + self._pool = GreenPool(size_or_pool) + size = size_or_pool self._responses = eventlet.queue.LightQueue(size) self._inflight = 0 @@ -2646,6 +2651,10 @@ def public(func): def quorum_size(n): """ + quorum size as it applies to services that use 'replication' for data + integrity (Account/Container services). Object quorum_size is defined + on a storage policy basis. + Number of successful backend requests needed for the proxy to consider the client request successful. """ @@ -3139,6 +3148,26 @@ _rfc_extension_pattern = re.compile( r'(?:\s*;\s*(' + _rfc_token + r")\s*(?:=\s*(" + _rfc_token + r'|"(?:[^"\\]|\\.)*"))?)') +_content_range_pattern = re.compile(r'^bytes (\d+)-(\d+)/(\d+)$') + + +def parse_content_range(content_range): + """ + Parse a content-range header into (first_byte, last_byte, total_size). + + See RFC 7233 section 4.2 for details on the header format, but it's + basically "Content-Range: bytes ${start}-${end}/${total}". + + :param content_range: Content-Range header value to parse, + e.g. "bytes 100-1249/49004" + :returns: 3-tuple (start, end, total) + :raises: ValueError if malformed + """ + found = re.search(_content_range_pattern, content_range) + if not found: + raise ValueError("malformed Content-Range %r" % (content_range,)) + return tuple(int(x) for x in found.groups()) + def parse_content_type(content_type): """ @@ -3293,8 +3322,11 @@ def iter_multipart_mime_documents(wsgi_input, boundary, read_chunk_size=4096): :raises: MimeInvalid if the document is malformed """ boundary = '--' + boundary - if wsgi_input.readline(len(boundary + '\r\n')).strip() != boundary: - raise swift.common.exceptions.MimeInvalid('invalid starting boundary') + blen = len(boundary) + 2 # \r\n + got = wsgi_input.readline(blen) + if got.strip() != boundary: + raise swift.common.exceptions.MimeInvalid( + 'invalid starting boundary: wanted %r, got %r', (boundary, got)) boundary = '\r\n' + boundary input_buffer = '' done = False diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index b1e1f5ea7..35df2077f 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -25,6 +25,7 @@ import time import mimetools from swift import gettext_ as _ from StringIO import StringIO +from textwrap import dedent import eventlet import eventlet.debug @@ -96,13 +97,34 @@ def _loadconfigdir(object_type, uri, path, name, relative_to, global_conf): loadwsgi._loaders['config_dir'] = _loadconfigdir +class ConfigString(NamedConfigLoader): + """ + Wrap a raw config string up for paste.deploy. + + If you give one of these to our loadcontext (e.g. give it to our + appconfig) we'll intercept it and get it routed to the right loader. + """ + + def __init__(self, config_string): + self.contents = StringIO(dedent(config_string)) + self.filename = "string" + defaults = { + 'here': "string", + '__file__': "string", + } + self.parser = loadwsgi.NicerConfigParser("string", defaults=defaults) + self.parser.optionxform = str # Don't lower-case keys + self.parser.readfp(self.contents) + + def wrap_conf_type(f): """ Wrap a function whos first argument is a paste.deploy style config uri, - such that you can pass it an un-adorned raw filesystem path and the config - directive (either config: or config_dir:) will be added automatically - based on the type of filesystem entity at the given path (either a file or - directory) before passing it through to the paste.deploy function. + such that you can pass it an un-adorned raw filesystem path (or config + string) and the config directive (either config:, config_dir:, or + config_str:) will be added automatically based on the type of entity + (either a file or directory, or if no such entity on the file system - + just a string) before passing it through to the paste.deploy function. """ def wrapper(conf_path, *args, **kwargs): if os.path.isdir(conf_path): @@ -332,6 +354,12 @@ class PipelineWrapper(object): def loadcontext(object_type, uri, name=None, relative_to=None, global_conf=None): + if isinstance(uri, loadwsgi.ConfigLoader): + # bypass loadcontext's uri parsing and loader routing and + # just directly return the context + if global_conf: + uri.update_defaults(global_conf, overwrite=False) + return uri.get_context(object_type, name, global_conf) add_conf_type = wrap_conf_type(lambda x: x) return loadwsgi.loadcontext(object_type, add_conf_type(uri), name=name, relative_to=relative_to, diff --git a/swift/container/sync.py b/swift/container/sync.py index 0f42de6e9..a409de4ac 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import errno import os import uuid from swift import gettext_ as _ @@ -25,8 +26,8 @@ from eventlet import sleep, Timeout import swift.common.db from swift.container.backend import ContainerBroker, DATADIR from swift.common.container_sync_realms import ContainerSyncRealms -from swift.common.direct_client import direct_get_object -from swift.common.internal_client import delete_object, put_object +from swift.common.internal_client import ( + delete_object, put_object, InternalClient, UnexpectedResponse) from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.ring.utils import is_local_device @@ -37,6 +38,55 @@ from swift.common.utils import ( from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND from swift.common.storage_policy import POLICIES +from swift.common.wsgi import ConfigString + + +# The default internal client config body is to support upgrades without +# requiring deployment of the new /etc/swift/internal-client.conf +ic_conf_body = """ +[DEFAULT] +# swift_dir = /etc/swift +# user = swift +# You can specify default log routing here if you want: +# log_name = swift +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# +# comma separated list of functions to call to setup custom log handlers. +# functions get passed: conf, name, log_to_console, log_route, fmt, logger, +# adapted_logger +# log_custom_handlers = +# +# If set, log_udp_host will override log_address +# log_udp_host = +# log_udp_port = 514 +# +# You can enable StatsD logging here: +# log_statsd_host = localhost +# log_statsd_port = 8125 +# log_statsd_default_sample_rate = 1.0 +# log_statsd_sample_rate_factor = 1.0 +# log_statsd_metric_prefix = + +[pipeline:main] +pipeline = catch_errors proxy-logging cache proxy-server + +[app:proxy-server] +use = egg:swift#proxy +# See proxy-server.conf-sample for options + +[filter:cache] +use = egg:swift#memcache +# See proxy-server.conf-sample for options + +[filter:proxy-logging] +use = egg:swift#proxy_logging + +[filter:catch_errors] +use = egg:swift#catch_errors +# See proxy-server.conf-sample for options +""".lstrip() class ContainerSync(Daemon): @@ -103,12 +153,12 @@ class ContainerSync(Daemon): loaded. This is overridden by unit tests. """ - def __init__(self, conf, container_ring=None): + def __init__(self, conf, container_ring=None, logger=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. - self.logger = get_logger(conf, log_route='container-sync') + self.logger = logger or get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount @@ -159,6 +209,26 @@ class ContainerSync(Daemon): swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.conn_timeout = float(conf.get('conn_timeout', 5)) + request_tries = int(conf.get('request_tries') or 3) + + internal_client_conf_path = conf.get('internal_client_conf_path') + if not internal_client_conf_path: + self.logger.warning( + _('Configuration option internal_client_conf_path not ' + 'defined. Using default configuration, See ' + 'internal-client.conf-sample for options')) + internal_client_conf = ConfigString(ic_conf_body) + else: + internal_client_conf = internal_client_conf_path + try: + self.swift = InternalClient( + internal_client_conf, 'Swift Container Sync', request_tries) + except IOError as err: + if err.errno != errno.ENOENT: + raise + raise SystemExit( + _('Unable to load internal client from config: %r (%s)') % + (internal_client_conf_path, err)) def get_object_ring(self, policy_idx): """ @@ -380,39 +450,32 @@ class ContainerSync(Daemon): looking_for_timestamp = Timestamp(row['created_at']) timestamp = -1 headers = body = None - headers_out = {'X-Backend-Storage-Policy-Index': + # look up for the newest one + headers_out = {'X-Newest': True, + 'X-Backend-Storage-Policy-Index': str(info['storage_policy_index'])} - for node in nodes: - try: - these_headers, this_body = direct_get_object( - node, part, info['account'], info['container'], - row['name'], headers=headers_out, - resp_chunk_size=65536) - this_timestamp = Timestamp( - these_headers['x-timestamp']) - if this_timestamp > timestamp: - timestamp = this_timestamp - headers = these_headers - body = this_body - except ClientException as err: - # If any errors are not 404, make sure we report the - # non-404 one. We don't want to mistakenly assume the - # object no longer exists just because one says so and - # the others errored for some other reason. - if not exc or getattr( - exc, 'http_status', HTTP_NOT_FOUND) == \ - HTTP_NOT_FOUND: - exc = err - except (Exception, Timeout) as err: - exc = err + try: + source_obj_status, source_obj_info, source_obj_iter = \ + self.swift.get_object(info['account'], + info['container'], row['name'], + headers=headers_out, + acceptable_statuses=(2, 4)) + + except (Exception, UnexpectedResponse, Timeout) as err: + source_obj_info = {} + source_obj_iter = None + exc = err + timestamp = Timestamp(source_obj_info.get( + 'x-timestamp', 0)) + headers = source_obj_info + body = source_obj_iter if timestamp < looking_for_timestamp: if exc: raise exc raise Exception( - _('Unknown exception trying to GET: %(node)r ' + _('Unknown exception trying to GET: ' '%(account)r %(container)r %(object)r'), - {'node': node, 'part': part, - 'account': info['account'], + {'account': info['account'], 'container': info['container'], 'object': row['name']}) for key in ('date', 'last-modified'): diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index a8d14dfa2..c49d557fe 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -40,7 +40,7 @@ import hashlib import logging import traceback import xattr -from os.path import basename, dirname, exists, getmtime, join +from os.path import basename, dirname, exists, getmtime, join, splitext from random import shuffle from tempfile import mkstemp from contextlib import contextmanager @@ -50,7 +50,7 @@ from eventlet import Timeout from eventlet.hubs import trampoline from swift import gettext_ as _ -from swift.common.constraints import check_mount +from swift.common.constraints import check_mount, check_dir from swift.common.request_helpers import is_sys_meta from swift.common.utils import mkdirs, Timestamp, \ storage_directory, hash_path, renamer, fallocate, fsync, \ @@ -63,7 +63,9 @@ from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \ DiskFileDeleted, DiskFileError, DiskFileNotOpen, PathNotDir, \ ReplicationLockTimeout, DiskFileExpired, DiskFileXattrNotSupported from swift.common.swob import multi_range_iterator -from swift.common.storage_policy import get_policy_string, POLICIES +from swift.common.storage_policy import ( + get_policy_string, split_policy_string, PolicyError, POLICIES, + REPL_POLICY, EC_POLICY) from functools import partial @@ -154,10 +156,10 @@ def write_metadata(fd, metadata, xattr_size=65536): raise -def extract_policy_index(obj_path): +def extract_policy(obj_path): """ - Extracts the policy index for an object (based on the name of the objects - directory) given the device-relative path to the object. Returns 0 in + Extracts the policy for an object (based on the name of the objects + directory) given the device-relative path to the object. Returns None in the event that the path is malformed in some way. The device-relative path is everything after the mount point; for example: @@ -170,19 +172,18 @@ def extract_policy_index(obj_path): objects-5/179/485dc017205a81df3af616d917c90179/1401811134.873649.data :param obj_path: device-relative path of an object - :returns: storage policy index + :returns: a :class:`~swift.common.storage_policy.BaseStoragePolicy` or None """ - policy_idx = 0 try: obj_portion = obj_path[obj_path.index(DATADIR_BASE):] obj_dirname = obj_portion[:obj_portion.index('/')] except Exception: - return policy_idx - if '-' in obj_dirname: - base, policy_idx = obj_dirname.split('-', 1) - if POLICIES.get_by_index(policy_idx) is None: - policy_idx = 0 - return int(policy_idx) + return None + try: + base, policy = split_policy_string(obj_dirname) + except PolicyError: + return None + return policy def quarantine_renamer(device_path, corrupted_file_path): @@ -197,9 +198,13 @@ def quarantine_renamer(device_path, corrupted_file_path): :raises OSError: re-raises non errno.EEXIST / errno.ENOTEMPTY exceptions from rename """ + policy = extract_policy(corrupted_file_path) + if policy is None: + # TODO: support a quarantine-unknown location + policy = POLICIES.legacy from_dir = dirname(corrupted_file_path) to_dir = join(device_path, 'quarantined', - get_data_dir(extract_policy_index(corrupted_file_path)), + get_data_dir(policy), basename(from_dir)) invalidate_hash(dirname(from_dir)) try: @@ -429,8 +434,9 @@ class AuditLocation(object): stringify to a filesystem path so the auditor's logs look okay. """ - def __init__(self, path, device, partition): - self.path, self.device, self.partition = path, device, partition + def __init__(self, path, device, partition, policy): + self.path, self.device, self.partition, self.policy = ( + path, device, partition, policy) def __str__(self): return str(self.path) @@ -470,19 +476,17 @@ def object_audit_location_generator(devices, mount_check=True, logger=None, _('Skipping %s as it is not mounted'), device) continue # loop through object dirs for all policies - for dir in [dir for dir in os.listdir(os.path.join(devices, device)) - if dir.startswith(DATADIR_BASE)]: - datadir_path = os.path.join(devices, device, dir) - # warn if the object dir doesn't match with a policy - policy_idx = 0 - if '-' in dir: - base, policy_idx = dir.split('-', 1) + for dir_ in os.listdir(os.path.join(devices, device)): + if not dir_.startswith(DATADIR_BASE): + continue try: - get_data_dir(policy_idx) - except ValueError: + base, policy = split_policy_string(dir_) + except PolicyError as e: if logger: - logger.warn(_('Directory %s does not map to a ' - 'valid policy') % dir) + logger.warn(_('Directory %r does not map ' + 'to a valid policy (%s)') % (dir_, e)) + continue + datadir_path = os.path.join(devices, device, dir_) partitions = listdir(datadir_path) for partition in partitions: part_path = os.path.join(datadir_path, partition) @@ -502,9 +506,50 @@ def object_audit_location_generator(devices, mount_check=True, logger=None, continue for hsh in hashes: hsh_path = os.path.join(suff_path, hsh) - yield AuditLocation(hsh_path, device, partition) + yield AuditLocation(hsh_path, device, partition, + policy) + + +def strip_self(f): + """ + Wrapper to attach module level functions to base class. + """ + def wrapper(self, *args, **kwargs): + return f(*args, **kwargs) + return wrapper + +class DiskFileRouter(object): + policy_type_to_manager_cls = {} + + @classmethod + def register(cls, policy_type): + """ + Decorator for Storage Policy implementations to register + their DiskFile implementation. + """ + def register_wrapper(diskfile_cls): + if policy_type in cls.policy_type_to_manager_cls: + raise PolicyError( + '%r is already registered for the policy_type %r' % ( + cls.policy_type_to_manager_cls[policy_type], + policy_type)) + cls.policy_type_to_manager_cls[policy_type] = diskfile_cls + return diskfile_cls + return register_wrapper + + def __init__(self, *args, **kwargs): + self.policy_to_manager = {} + for policy in POLICIES: + manager_cls = self.policy_type_to_manager_cls[policy.policy_type] + self.policy_to_manager[policy] = manager_cls(*args, **kwargs) + + def __getitem__(self, policy): + return self.policy_to_manager[policy] + + +@DiskFileRouter.register(REPL_POLICY) class DiskFileManager(object): """ Management class for devices, providing common place for shared parameters @@ -527,6 +572,16 @@ class DiskFileManager(object): :param conf: caller provided configuration object :param logger: caller provided logger """ + + diskfile_cls = None # DiskFile will be set after that class is defined + + # module level functions dropped to implementation specific + hash_cleanup_listdir = strip_self(hash_cleanup_listdir) + _get_hashes = strip_self(get_hashes) + invalidate_hash = strip_self(invalidate_hash) + get_ondisk_files = strip_self(get_ondisk_files) + quarantine_renamer = strip_self(quarantine_renamer) + def __init__(self, conf, logger): self.logger = logger self.devices = conf.get('devices', '/srv/node') @@ -583,21 +638,25 @@ class DiskFileManager(object): def get_dev_path(self, device, mount_check=None): """ - Return the path to a device, checking to see that it is a proper mount - point based on a configuration parameter. + Return the path to a device, first checking to see if either it + is a proper mount point, or at least a directory depending on + the mount_check configuration option. :param device: name of target device :param mount_check: whether or not to check mountedness of device. Defaults to bool(self.mount_check). :returns: full path to the device, None if the path to the device is - not a proper mount point. + not a proper mount point or directory. """ - should_check = self.mount_check if mount_check is None else mount_check - if should_check and not check_mount(self.devices, device): - dev_path = None - else: - dev_path = os.path.join(self.devices, device) - return dev_path + # we'll do some kind of check unless explicitly forbidden + if mount_check is not False: + if mount_check or self.mount_check: + check = check_mount + else: + check = check_dir + if not check(self.devices, device): + return None + return os.path.join(self.devices, device) @contextmanager def replication_lock(self, device): @@ -619,28 +678,27 @@ class DiskFileManager(object): yield True def pickle_async_update(self, device, account, container, obj, data, - timestamp, policy_idx): + timestamp, policy): device_path = self.construct_dev_path(device) - async_dir = os.path.join(device_path, get_async_dir(policy_idx)) + async_dir = os.path.join(device_path, get_async_dir(policy)) ohash = hash_path(account, container, obj) self.threadpools[device].run_in_thread( write_pickle, data, os.path.join(async_dir, ohash[-3:], ohash + '-' + Timestamp(timestamp).internal), - os.path.join(device_path, get_tmp_dir(policy_idx))) + os.path.join(device_path, get_tmp_dir(policy))) self.logger.increment('async_pendings') def get_diskfile(self, device, partition, account, container, obj, - policy_idx=0, **kwargs): + policy, **kwargs): dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() - return DiskFile(self, dev_path, self.threadpools[device], - partition, account, container, obj, - policy_idx=policy_idx, - use_splice=self.use_splice, pipe_size=self.pipe_size, - **kwargs) + return self.diskfile_cls(self, dev_path, self.threadpools[device], + partition, account, container, obj, + policy=policy, use_splice=self.use_splice, + pipe_size=self.pipe_size, **kwargs) def object_audit_location_generator(self, device_dirs=None): return object_audit_location_generator(self.devices, self.mount_check, @@ -648,12 +706,12 @@ class DiskFileManager(object): def get_diskfile_from_audit_location(self, audit_location): dev_path = self.get_dev_path(audit_location.device, mount_check=False) - return DiskFile.from_hash_dir( + return self.diskfile_cls.from_hash_dir( self, audit_location.path, dev_path, - audit_location.partition) + audit_location.partition, policy=audit_location.policy) def get_diskfile_from_hash(self, device, partition, object_hash, - policy_idx, **kwargs): + policy, **kwargs): """ Returns a DiskFile instance for an object at the given object_hash. Just in case someone thinks of refactoring, be @@ -667,13 +725,14 @@ class DiskFileManager(object): if not dev_path: raise DiskFileDeviceUnavailable() object_path = os.path.join( - dev_path, get_data_dir(policy_idx), partition, object_hash[-3:], + dev_path, get_data_dir(policy), str(partition), object_hash[-3:], object_hash) try: - filenames = hash_cleanup_listdir(object_path, self.reclaim_age) + filenames = self.hash_cleanup_listdir(object_path, + self.reclaim_age) except OSError as err: if err.errno == errno.ENOTDIR: - quar_path = quarantine_renamer(dev_path, object_path) + quar_path = self.quarantine_renamer(dev_path, object_path) logging.exception( _('Quarantined %(object_path)s to %(quar_path)s because ' 'it is not a directory'), {'object_path': object_path, @@ -693,21 +752,20 @@ class DiskFileManager(object): metadata.get('name', ''), 3, 3, True) except ValueError: raise DiskFileNotExist() - return DiskFile(self, dev_path, self.threadpools[device], - partition, account, container, obj, - policy_idx=policy_idx, **kwargs) + return self.diskfile_cls(self, dev_path, self.threadpools[device], + partition, account, container, obj, + policy=policy, **kwargs) - def get_hashes(self, device, partition, suffix, policy_idx): + def get_hashes(self, device, partition, suffixes, policy): dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() - partition_path = os.path.join(dev_path, get_data_dir(policy_idx), + partition_path = os.path.join(dev_path, get_data_dir(policy), partition) if not os.path.exists(partition_path): mkdirs(partition_path) - suffixes = suffix.split('-') if suffix else [] _junk, hashes = self.threadpools[device].force_run_in_thread( - get_hashes, partition_path, recalculate=suffixes) + self._get_hashes, partition_path, recalculate=suffixes) return hashes def _listdir(self, path): @@ -720,7 +778,7 @@ class DiskFileManager(object): path, err) return [] - def yield_suffixes(self, device, partition, policy_idx): + def yield_suffixes(self, device, partition, policy): """ Yields tuples of (full_path, suffix_only) for suffixes stored on the given device and partition. @@ -728,7 +786,7 @@ class DiskFileManager(object): dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() - partition_path = os.path.join(dev_path, get_data_dir(policy_idx), + partition_path = os.path.join(dev_path, get_data_dir(policy), partition) for suffix in self._listdir(partition_path): if len(suffix) != 3: @@ -739,7 +797,7 @@ class DiskFileManager(object): continue yield (os.path.join(partition_path, suffix), suffix) - def yield_hashes(self, device, partition, policy_idx, suffixes=None): + def yield_hashes(self, device, partition, policy, suffixes=None, **kwargs): """ Yields tuples of (full_path, hash_only, timestamp) for object information stored for the given device, partition, and @@ -752,17 +810,18 @@ class DiskFileManager(object): if not dev_path: raise DiskFileDeviceUnavailable() if suffixes is None: - suffixes = self.yield_suffixes(device, partition, policy_idx) + suffixes = self.yield_suffixes(device, partition, policy) else: - partition_path = os.path.join(dev_path, get_data_dir(policy_idx), - partition) + partition_path = os.path.join(dev_path, + get_data_dir(policy), + str(partition)) suffixes = ( (os.path.join(partition_path, suffix), suffix) for suffix in suffixes) for suffix_path, suffix in suffixes: for object_hash in self._listdir(suffix_path): object_path = os.path.join(suffix_path, object_hash) - for name in hash_cleanup_listdir( + for name in self.hash_cleanup_listdir( object_path, self.reclaim_age): ts, ext = name.rsplit('.', 1) yield (object_path, object_hash, ts) @@ -794,8 +853,11 @@ class DiskFileWriter(object): :param tmppath: full path name of the opened file descriptor :param bytes_per_sync: number bytes written between sync calls :param threadpool: internal thread pool to use for disk operations + :param diskfile: the diskfile creating this DiskFileWriter instance """ - def __init__(self, name, datadir, fd, tmppath, bytes_per_sync, threadpool): + + def __init__(self, name, datadir, fd, tmppath, bytes_per_sync, threadpool, + diskfile): # Parameter tracking self._name = name self._datadir = datadir @@ -803,6 +865,7 @@ class DiskFileWriter(object): self._tmppath = tmppath self._bytes_per_sync = bytes_per_sync self._threadpool = threadpool + self._diskfile = diskfile # Internal attributes self._upload_size = 0 @@ -811,6 +874,10 @@ class DiskFileWriter(object): self._put_succeeded = False @property + def manager(self): + return self._diskfile.manager + + @property def put_succeeded(self): return self._put_succeeded @@ -855,7 +922,7 @@ class DiskFileWriter(object): # drop_cache() after fsync() to avoid redundant work (pages all # clean). drop_buffer_cache(self._fd, 0, self._upload_size) - invalidate_hash(dirname(self._datadir)) + self.manager.invalidate_hash(dirname(self._datadir)) # After the rename completes, this object will be available for other # requests to reference. renamer(self._tmppath, target_path) @@ -864,7 +931,7 @@ class DiskFileWriter(object): # succeeded, the tempfile would no longer exist at its original path. self._put_succeeded = True try: - hash_cleanup_listdir(self._datadir) + self.manager.hash_cleanup_listdir(self._datadir) except OSError: logging.exception(_('Problem cleaning up %s'), self._datadir) @@ -887,6 +954,16 @@ class DiskFileWriter(object): self._threadpool.force_run_in_thread( self._finalize_put, metadata, target_path) + def commit(self, timestamp): + """ + Perform any operations necessary to mark the object as durable. For + replication policy type this is a no-op. + + :param timestamp: object put timestamp, an instance of + :class:`~swift.common.utils.Timestamp` + """ + pass + class DiskFileReader(object): """ @@ -917,17 +994,20 @@ class DiskFileReader(object): :param quarantine_hook: 1-arg callable called w/reason when quarantined :param use_splice: if true, use zero-copy splice() to send data :param pipe_size: size of pipe buffer used in zero-copy operations + :param diskfile: the diskfile creating this DiskFileReader instance :param keep_cache: should resulting reads be kept in the buffer cache """ def __init__(self, fp, data_file, obj_size, etag, threadpool, disk_chunk_size, keep_cache_size, device_path, logger, - quarantine_hook, use_splice, pipe_size, keep_cache=False): + quarantine_hook, use_splice, pipe_size, diskfile, + keep_cache=False): # Parameter tracking self._fp = fp self._data_file = data_file self._obj_size = obj_size self._etag = etag self._threadpool = threadpool + self._diskfile = diskfile self._disk_chunk_size = disk_chunk_size self._device_path = device_path self._logger = logger @@ -950,6 +1030,10 @@ class DiskFileReader(object): self._suppress_file_closing = False self._quarantined_dir = None + @property + def manager(self): + return self._diskfile.manager + def __iter__(self): """Returns an iterator over the data file.""" try: @@ -1130,7 +1214,8 @@ class DiskFileReader(object): def _quarantine(self, msg): self._quarantined_dir = self._threadpool.run_in_thread( - quarantine_renamer, self._device_path, self._data_file) + self.manager.quarantine_renamer, self._device_path, + self._data_file) self._logger.warn("Quarantined object %s: %s" % ( self._data_file, msg)) self._logger.increment('quarantines') @@ -1196,15 +1281,18 @@ class DiskFile(object): :param container: container name for the object :param obj: object name for the object :param _datadir: override the full datadir otherwise constructed here - :param policy_idx: used to get the data dir when constructing it here + :param policy: the StoragePolicy instance :param use_splice: if true, use zero-copy splice() to send data :param pipe_size: size of pipe buffer used in zero-copy operations """ + reader_cls = DiskFileReader + writer_cls = DiskFileWriter + def __init__(self, mgr, device_path, threadpool, partition, account=None, container=None, obj=None, _datadir=None, - policy_idx=0, use_splice=False, pipe_size=None): - self._mgr = mgr + policy=None, use_splice=False, pipe_size=None, **kwargs): + self._manager = mgr self._device_path = device_path self._threadpool = threadpool or ThreadPool(nthreads=0) self._logger = mgr.logger @@ -1212,6 +1300,7 @@ class DiskFile(object): self._bytes_per_sync = mgr.bytes_per_sync self._use_splice = use_splice self._pipe_size = pipe_size + self.policy = policy if account and container and obj: self._name = '/' + '/'.join((account, container, obj)) self._account = account @@ -1219,7 +1308,7 @@ class DiskFile(object): self._obj = obj name_hash = hash_path(account, container, obj) self._datadir = join( - device_path, storage_directory(get_data_dir(policy_idx), + device_path, storage_directory(get_data_dir(policy), partition, name_hash)) else: # gets populated when we read the metadata @@ -1228,7 +1317,7 @@ class DiskFile(object): self._container = None self._obj = None self._datadir = None - self._tmpdir = join(device_path, get_tmp_dir(policy_idx)) + self._tmpdir = join(device_path, get_tmp_dir(policy)) self._metadata = None self._data_file = None self._fp = None @@ -1239,10 +1328,14 @@ class DiskFile(object): else: name_hash = hash_path(account, container, obj) self._datadir = join( - device_path, storage_directory(get_data_dir(policy_idx), + device_path, storage_directory(get_data_dir(policy), partition, name_hash)) @property + def manager(self): + return self._manager + + @property def account(self): return self._account @@ -1267,8 +1360,9 @@ class DiskFile(object): return Timestamp(self._metadata.get('X-Timestamp')) @classmethod - def from_hash_dir(cls, mgr, hash_dir_path, device_path, partition): - return cls(mgr, device_path, None, partition, _datadir=hash_dir_path) + def from_hash_dir(cls, mgr, hash_dir_path, device_path, partition, policy): + return cls(mgr, device_path, None, partition, _datadir=hash_dir_path, + policy=policy) def open(self): """ @@ -1307,7 +1401,7 @@ class DiskFile(object): .. note:: - An implemenation shall raise `DiskFileNotOpen` when has not + An implementation shall raise `DiskFileNotOpen` when has not previously invoked the :func:`swift.obj.diskfile.DiskFile.open` method. """ @@ -1339,7 +1433,7 @@ class DiskFile(object): :returns: DiskFileQuarantined exception object """ self._quarantined_dir = self._threadpool.run_in_thread( - quarantine_renamer, self._device_path, data_file) + self.manager.quarantine_renamer, self._device_path, data_file) self._logger.warn("Quarantined object %s: %s" % ( data_file, msg)) self._logger.increment('quarantines') @@ -1384,7 +1478,7 @@ class DiskFile(object): # The data directory does not exist, so the object cannot exist. fileset = (None, None, None) else: - fileset = get_ondisk_files(files, self._datadir) + fileset = self.manager.get_ondisk_files(files, self._datadir) return fileset def _construct_exception_from_ts_file(self, ts_file): @@ -1576,12 +1670,12 @@ class DiskFile(object): Not needed by the REST layer. :returns: a :class:`swift.obj.diskfile.DiskFileReader` object """ - dr = DiskFileReader( + dr = self.reader_cls( self._fp, self._data_file, int(self._metadata['Content-Length']), self._metadata['ETag'], self._threadpool, self._disk_chunk_size, - self._mgr.keep_cache_size, self._device_path, self._logger, + self._manager.keep_cache_size, self._device_path, self._logger, use_splice=self._use_splice, quarantine_hook=_quarantine_hook, - pipe_size=self._pipe_size, keep_cache=keep_cache) + pipe_size=self._pipe_size, diskfile=self, keep_cache=keep_cache) # At this point the reader object is now responsible for closing # the file pointer. self._fp = None @@ -1621,8 +1715,10 @@ class DiskFile(object): if err.errno in (errno.ENOSPC, errno.EDQUOT): raise DiskFileNoSpace() raise - dfw = DiskFileWriter(self._name, self._datadir, fd, tmppath, - self._bytes_per_sync, self._threadpool) + dfw = self.writer_cls(self._name, self._datadir, fd, tmppath, + bytes_per_sync=self._bytes_per_sync, + threadpool=self._threadpool, + diskfile=self) yield dfw finally: try: @@ -1671,8 +1767,619 @@ class DiskFile(object): :raises DiskFileError: this implementation will raise the same errors as the `create()` method. """ - timestamp = Timestamp(timestamp).internal - + # this is dumb, only tests send in strings + timestamp = Timestamp(timestamp) with self.create() as deleter: deleter._extension = '.ts' - deleter.put({'X-Timestamp': timestamp}) + deleter.put({'X-Timestamp': timestamp.internal}) + +# TODO: move DiskFileManager definition down here +DiskFileManager.diskfile_cls = DiskFile + + +class ECDiskFileReader(DiskFileReader): + pass + + +class ECDiskFileWriter(DiskFileWriter): + + def _finalize_durable(self, durable_file_path): + exc = msg = None + try: + with open(durable_file_path, 'w') as _fd: + fsync(_fd) + try: + self.manager.hash_cleanup_listdir(self._datadir) + except OSError: + self.manager.logger.exception( + _('Problem cleaning up %s'), self._datadir) + except OSError: + msg = (_('Problem fsyncing durable state file: %s'), + durable_file_path) + exc = DiskFileError(msg) + except IOError as io_err: + if io_err.errno in (errno.ENOSPC, errno.EDQUOT): + msg = (_("No space left on device for %s"), + durable_file_path) + exc = DiskFileNoSpace() + else: + msg = (_('Problem writing durable state file: %s'), + durable_file_path) + exc = DiskFileError(msg) + if exc: + self.manager.logger.exception(msg) + raise exc + + def commit(self, timestamp): + """ + Finalize put by writing a timestamp.durable file for the object. We + do this for EC policy because it requires a 2-phase put commit + confirmation. + + :param timestamp: object put timestamp, an instance of + :class:`~swift.common.utils.Timestamp` + """ + durable_file_path = os.path.join( + self._datadir, timestamp.internal + '.durable') + self._threadpool.force_run_in_thread( + self._finalize_durable, durable_file_path) + + def put(self, metadata): + """ + The only difference between this method and the replication policy + DiskFileWriter method is the call into manager.make_on_disk_filename + to construct the data file name. + """ + timestamp = Timestamp(metadata['X-Timestamp']) + fi = None + if self._extension == '.data': + # generally we treat the fragment index provided in metadata as + # canon, but if it's unavailable (e.g. tests) it's reasonable to + # use the frag_index provided at instantiation. Either way make + # sure that the fragment index is included in object sysmeta. + fi = metadata.setdefault('X-Object-Sysmeta-Ec-Frag-Index', + self._diskfile._frag_index) + filename = self.manager.make_on_disk_filename( + timestamp, self._extension, frag_index=fi) + metadata['name'] = self._name + target_path = join(self._datadir, filename) + + self._threadpool.force_run_in_thread( + self._finalize_put, metadata, target_path) + + +class ECDiskFile(DiskFile): + + reader_cls = ECDiskFileReader + writer_cls = ECDiskFileWriter + + def __init__(self, *args, **kwargs): + super(ECDiskFile, self).__init__(*args, **kwargs) + frag_index = kwargs.get('frag_index') + self._frag_index = None + if frag_index is not None: + self._frag_index = self.manager.validate_fragment_index(frag_index) + + def _get_ondisk_file(self): + """ + The only difference between this method and the replication policy + DiskFile method is passing in the frag_index kwarg to our manager's + get_ondisk_files method. + """ + try: + files = os.listdir(self._datadir) + except OSError as err: + if err.errno == errno.ENOTDIR: + # If there's a file here instead of a directory, quarantine + # it; something's gone wrong somewhere. + raise self._quarantine( + # hack: quarantine_renamer actually renames the directory + # enclosing the filename you give it, but here we just + # want this one file and not its parent. + os.path.join(self._datadir, "made-up-filename"), + "Expected directory, found file at %s" % self._datadir) + elif err.errno != errno.ENOENT: + raise DiskFileError( + "Error listing directory %s: %s" % (self._datadir, err)) + # The data directory does not exist, so the object cannot exist. + fileset = (None, None, None) + else: + fileset = self.manager.get_ondisk_files( + files, self._datadir, frag_index=self._frag_index) + return fileset + + def purge(self, timestamp, frag_index): + """ + Remove a tombstone file matching the specified timestamp or + datafile matching the specified timestamp and fragment index + from the object directory. + + This provides the EC reconstructor/ssync process with a way to + remove a tombstone or fragment from a handoff node after + reverting it to its primary node. + + The hash will be invalidated, and if empty or invalid the + hsh_path will be removed on next hash_cleanup_listdir. + + :param timestamp: the object timestamp, an instance of + :class:`~swift.common.utils.Timestamp` + :param frag_index: a fragment archive index, must be a whole number. + """ + for ext in ('.data', '.ts'): + purge_file = self.manager.make_on_disk_filename( + timestamp, ext=ext, frag_index=frag_index) + remove_file(os.path.join(self._datadir, purge_file)) + self.manager.invalidate_hash(dirname(self._datadir)) + + +@DiskFileRouter.register(EC_POLICY) +class ECDiskFileManager(DiskFileManager): + diskfile_cls = ECDiskFile + + def validate_fragment_index(self, frag_index): + """ + Return int representation of frag_index, or raise a DiskFileError if + frag_index is not a whole number. + """ + try: + frag_index = int(str(frag_index)) + except (ValueError, TypeError) as e: + raise DiskFileError( + 'Bad fragment index: %s: %s' % (frag_index, e)) + if frag_index < 0: + raise DiskFileError( + 'Fragment index must not be negative: %s' % frag_index) + return frag_index + + def make_on_disk_filename(self, timestamp, ext=None, frag_index=None, + *a, **kw): + """ + Returns the EC specific filename for given timestamp. + + :param timestamp: the object timestamp, an instance of + :class:`~swift.common.utils.Timestamp` + :param ext: an optional string representing a file extension to be + appended to the returned file name + :param frag_index: a fragment archive index, used with .data extension + only, must be a whole number. + :returns: a file name + :raises DiskFileError: if ext=='.data' and the kwarg frag_index is not + a whole number + """ + rv = timestamp.internal + if ext == '.data': + # for datafiles only we encode the fragment index in the filename + # to allow archives of different indexes to temporarily be stored + # on the same node in certain situations + frag_index = self.validate_fragment_index(frag_index) + rv += '#' + str(frag_index) + if ext: + rv = '%s%s' % (rv, ext) + return rv + + def parse_on_disk_filename(self, filename): + """ + Returns the timestamp extracted from a policy specific .data file name. + For EC policy the data file name includes a fragment index which must + be stripped off to retrieve the timestamp. + + :param filename: the data file name including extension + :returns: a dict, with keys for timestamp, frag_index, and ext:: + + * timestamp is a :class:`~swift.common.utils.Timestamp` + * frag_index is an int or None + * ext is a string, the file extension including the leading dot or + the empty string if the filename has no extenstion. + + :raises DiskFileError: if any part of the filename is not able to be + validated. + """ + frag_index = None + filename, ext = splitext(filename) + parts = filename.split('#', 1) + timestamp = parts[0] + if ext == '.data': + # it is an error for an EC data file to not have a valid + # fragment index + try: + frag_index = parts[1] + except IndexError: + frag_index = None + frag_index = self.validate_fragment_index(frag_index) + return { + 'timestamp': Timestamp(timestamp), + 'frag_index': frag_index, + 'ext': ext, + } + + def is_obsolete(self, filename, other_filename): + """ + Test if a given file is considered to be obsolete with respect to + another file in an object storage dir. + + Implements EC policy specific behavior when comparing files against a + .durable file. + + A simple string comparison would consider t2#1.data to be older than + t2.durable (since t2#1.data < t2.durable). By stripping off the file + extensions we get the desired behavior: t2#1 > t2 without compromising + the detection of t1#1 < t2. + + :param filename: a string representing an absolute filename + :param other_filename: a string representing an absolute filename + :returns: True if filename is considered obsolete, False otherwise. + """ + if other_filename.endswith('.durable'): + return splitext(filename)[0] < splitext(other_filename)[0] + return filename < other_filename + + def _gather_on_disk_file(self, filename, ext, context, frag_index=None, + **kwargs): + """ + Called by gather_ondisk_files() for each file in an object + datadir in reverse sorted order. If a file is considered part of a + valid on-disk file set it will be added to the context dict, keyed by + its extension. If a file is considered to be obsolete it will be added + to a list stored under the key 'obsolete' in the context dict. + + :param filename: name of file to be accepted or not + :param ext: extension part of filename + :param context: a context dict that may have been populated by previous + calls to this method + :param frag_index: if set, search for a specific fragment index .data + file, otherwise accept the first valid .data file. + :returns: True if a valid file set has been found, False otherwise + """ + + # if first file with given extension then add filename to context + # dict and return True + accept_first = lambda: context.setdefault(ext, filename) == filename + # add the filename to the list of obsolete files in context dict + discard = lambda: context.setdefault('obsolete', []).append(filename) + # set a flag in the context dict indicating that a valid fileset has + # been found + set_valid_fileset = lambda: context.setdefault('found_valid', True) + # return True if the valid fileset flag is set in the context dict + have_valid_fileset = lambda: context.get('found_valid') + + if context.get('.durable'): + # a .durable file has been found + if ext == '.data': + if self.is_obsolete(filename, context.get('.durable')): + # this and remaining data files are older than durable + discard() + set_valid_fileset() + else: + # accept the first .data file if it matches requested + # frag_index, or if no specific frag_index is requested + fi = self.parse_on_disk_filename(filename)['frag_index'] + if frag_index is None or frag_index == int(fi): + accept_first() + set_valid_fileset() + # else: keep searching for a .data file to match frag_index + context.setdefault('fragments', []).append(filename) + else: + # there can no longer be a matching .data file so mark what has + # been found so far as the valid fileset + discard() + set_valid_fileset() + elif ext == '.data': + # not yet found a .durable + if have_valid_fileset(): + # valid fileset means we must have a newer + # .ts, so discard the older .data file + discard() + else: + # .data newer than a .durable or .ts, don't discard yet + context.setdefault('fragments_without_durable', []).append( + filename) + elif ext == '.ts': + if have_valid_fileset() or not accept_first(): + # newer .data, .durable or .ts already found so discard this + discard() + if not have_valid_fileset(): + # remove any .meta that may have been previously found + context['.meta'] = None + set_valid_fileset() + elif ext in ('.meta', '.durable'): + if have_valid_fileset() or not accept_first(): + # newer .data, .durable or .ts already found so discard this + discard() + else: + # ignore unexpected files + pass + return have_valid_fileset() + + def _verify_on_disk_files(self, accepted_files, frag_index=None, **kwargs): + """ + Verify that the final combination of on disk files complies with the + diskfile contract. + + :param accepted_files: files that have been found and accepted + :param frag_index: specifies a specific fragment index .data file + :returns: True if the file combination is compliant, False otherwise + """ + if not accepted_files.get('.data'): + # We may find only a .meta, which doesn't mean the on disk + # contract is broken. So we clear it to comply with + # superclass assertions. + accepted_files['.meta'] = None + + data_file, meta_file, ts_file, durable_file = tuple( + [accepted_files.get(ext) + for ext in ('.data', '.meta', '.ts', '.durable')]) + + return ((data_file is None or durable_file is not None) + and (data_file is None and meta_file is None + and ts_file is None and durable_file is None) + or (ts_file is not None and data_file is None + and meta_file is None and durable_file is None) + or (data_file is not None and durable_file is not None + and ts_file is None) + or (durable_file is not None and meta_file is None + and ts_file is None)) + + def gather_ondisk_files(self, files, include_obsolete=False, + frag_index=None, verify=False, **kwargs): + """ + Given a simple list of files names, iterate over them to determine the + files that constitute a valid object, and optionally determine the + files that are obsolete and could be deleted. Note that some files may + fall into neither category. + + :param files: a list of file names. + :param include_obsolete: By default the iteration will stop when a + valid file set has been found. Setting this + argument to True will cause the iteration to + continue in order to find all obsolete files. + :param frag_index: if set, search for a specific fragment index .data + file, otherwise accept the first valid .data file. + :returns: a dict that may contain: valid on disk files keyed by their + filename extension; a list of obsolete files stored under the + key 'obsolete'. + """ + # This visitor pattern enables future refactoring of other disk + # manager implementations to re-use this method and override + # _gather_ondisk_file and _verify_ondisk_files to apply implementation + # specific selection and verification of on-disk files. + files.sort(reverse=True) + results = {} + for afile in files: + ts_file = results.get('.ts') + data_file = results.get('.data') + if not include_obsolete: + assert ts_file is None, "On-disk file search loop" \ + " continuing after tombstone, %s, encountered" % ts_file + assert data_file is None, "On-disk file search loop" \ + " continuing after data file, %s, encountered" % data_file + + ext = splitext(afile)[1] + if self._gather_on_disk_file( + afile, ext, results, frag_index=frag_index, **kwargs): + if not include_obsolete: + break + + if verify: + assert self._verify_on_disk_files( + results, frag_index=frag_index, **kwargs), \ + "On-disk file search algorithm contract is broken: %s" \ + % results.values() + return results + + def get_ondisk_files(self, files, datadir, **kwargs): + """ + Given a simple list of files names, determine the files to use. + + :param files: simple set of files as a python list + :param datadir: directory name files are from for convenience + :returns: a tuple of data, meta, and tombstone + """ + # maintain compatibility with 'legacy' get_ondisk_files return value + accepted_files = self.gather_ondisk_files(files, verify=True, **kwargs) + result = [(join(datadir, accepted_files.get(ext)) + if accepted_files.get(ext) else None) + for ext in ('.data', '.meta', '.ts')] + return tuple(result) + + def cleanup_ondisk_files(self, hsh_path, reclaim_age=ONE_WEEK, + frag_index=None): + """ + Clean up on-disk files that are obsolete and gather the set of valid + on-disk files for an object. + + :param hsh_path: object hash path + :param reclaim_age: age in seconds at which to remove tombstones + :param frag_index: if set, search for a specific fragment index .data + file, otherwise accept the first valid .data file + :returns: a dict that may contain: valid on disk files keyed by their + filename extension; a list of obsolete files stored under the + key 'obsolete'; a list of files remaining in the directory, + reverse sorted, stored under the key 'files'. + """ + def is_reclaimable(filename): + timestamp = self.parse_on_disk_filename(filename)['timestamp'] + return (time.time() - float(timestamp)) > reclaim_age + + files = listdir(hsh_path) + files.sort(reverse=True) + results = self.gather_ondisk_files(files, include_obsolete=True, + frag_index=frag_index) + if '.durable' in results and not results.get('fragments'): + # a .durable with no .data is deleted as soon as it is found + results.setdefault('obsolete', []).append(results.pop('.durable')) + if '.ts' in results and is_reclaimable(results['.ts']): + results.setdefault('obsolete', []).append(results.pop('.ts')) + for filename in results.get('fragments_without_durable', []): + # stray fragments are not deleted until reclaim-age + if is_reclaimable(filename): + results.setdefault('obsolete', []).append(filename) + for filename in results.get('obsolete', []): + remove_file(join(hsh_path, filename)) + files.remove(filename) + results['files'] = files + return results + + def hash_cleanup_listdir(self, hsh_path, reclaim_age=ONE_WEEK): + """ + List contents of a hash directory and clean up any old files. + For EC policy, delete files older than a .durable or .ts file. + + :param hsh_path: object hash path + :param reclaim_age: age in seconds at which to remove tombstones + :returns: list of files remaining in the directory, reverse sorted + """ + # maintain compatibility with 'legacy' hash_cleanup_listdir + # return value + return self.cleanup_ondisk_files( + hsh_path, reclaim_age=reclaim_age)['files'] + + def yield_hashes(self, device, partition, policy, + suffixes=None, frag_index=None): + """ + This is the same as the replicated yield_hashes except when frag_index + is provided data files for fragment indexes not matching the given + frag_index are skipped. + """ + dev_path = self.get_dev_path(device) + if not dev_path: + raise DiskFileDeviceUnavailable() + if suffixes is None: + suffixes = self.yield_suffixes(device, partition, policy) + else: + partition_path = os.path.join(dev_path, + get_data_dir(policy), + str(partition)) + suffixes = ( + (os.path.join(partition_path, suffix), suffix) + for suffix in suffixes) + for suffix_path, suffix in suffixes: + for object_hash in self._listdir(suffix_path): + object_path = os.path.join(suffix_path, object_hash) + newest_valid_file = None + try: + results = self.cleanup_ondisk_files( + object_path, self.reclaim_age, frag_index=frag_index) + newest_valid_file = (results.get('.meta') + or results.get('.data') + or results.get('.ts')) + if newest_valid_file: + timestamp = self.parse_on_disk_filename( + newest_valid_file)['timestamp'] + yield (object_path, object_hash, timestamp.internal) + except AssertionError as err: + self.logger.debug('Invalid file set in %s (%s)' % ( + object_path, err)) + except DiskFileError as err: + self.logger.debug( + 'Invalid diskfile filename %r in %r (%s)' % ( + newest_valid_file, object_path, err)) + + def _hash_suffix(self, path, reclaim_age): + """ + The only difference between this method and the module level function + hash_suffix is the way that files are updated on the returned hash. + + Instead of all filenames hashed into a single hasher, each file name + will fall into a bucket either by fragment index for datafiles, or + None (indicating a durable, metadata or tombstone). + """ + # hash_per_fi instead of single hash for whole suffix + hash_per_fi = defaultdict(hashlib.md5) + try: + path_contents = sorted(os.listdir(path)) + except OSError as err: + if err.errno in (errno.ENOTDIR, errno.ENOENT): + raise PathNotDir() + raise + for hsh in path_contents: + hsh_path = join(path, hsh) + try: + files = self.hash_cleanup_listdir(hsh_path, reclaim_age) + except OSError as err: + if err.errno == errno.ENOTDIR: + partition_path = dirname(path) + objects_path = dirname(partition_path) + device_path = dirname(objects_path) + quar_path = quarantine_renamer(device_path, hsh_path) + logging.exception( + _('Quarantined %(hsh_path)s to %(quar_path)s because ' + 'it is not a directory'), {'hsh_path': hsh_path, + 'quar_path': quar_path}) + continue + raise + if not files: + try: + os.rmdir(hsh_path) + except OSError: + pass + # we just deleted this hsh_path, why are we waiting + # until the next suffix hash to raise PathNotDir so that + # this suffix will get del'd from the suffix hashes? + for filename in files: + info = self.parse_on_disk_filename(filename) + fi = info['frag_index'] + if fi is None: + hash_per_fi[fi].update(filename) + else: + hash_per_fi[fi].update(info['timestamp'].internal) + try: + os.rmdir(path) + except OSError: + pass + # here we flatten out the hashers hexdigest into a dictionary instead + # of just returning the one hexdigest for the whole suffix + return dict((fi, md5.hexdigest()) for fi, md5 in hash_per_fi.items()) + + def _get_hashes(self, partition_path, recalculate=None, do_listdir=False, + reclaim_age=None): + """ + The only difference with this method and the module level function + get_hashes is the call to hash_suffix routes to a method _hash_suffix + on this instance. + """ + reclaim_age = reclaim_age or self.reclaim_age + hashed = 0 + hashes_file = join(partition_path, HASH_FILE) + modified = False + force_rewrite = False + hashes = {} + mtime = -1 + + if recalculate is None: + recalculate = [] + + try: + with open(hashes_file, 'rb') as fp: + hashes = pickle.load(fp) + mtime = getmtime(hashes_file) + except Exception: + do_listdir = True + force_rewrite = True + if do_listdir: + for suff in os.listdir(partition_path): + if len(suff) == 3: + hashes.setdefault(suff, None) + modified = True + hashes.update((suffix, None) for suffix in recalculate) + for suffix, hash_ in hashes.items(): + if not hash_: + suffix_dir = join(partition_path, suffix) + try: + hashes[suffix] = self._hash_suffix(suffix_dir, reclaim_age) + hashed += 1 + except PathNotDir: + del hashes[suffix] + except OSError: + logging.exception(_('Error hashing suffix')) + modified = True + if modified: + with lock_path(partition_path): + if force_rewrite or not exists(hashes_file) or \ + getmtime(hashes_file) == mtime: + write_pickle( + hashes, hashes_file, partition_path, PICKLE_PROTOCOL) + return hashed, hashes + return self._get_hashes(partition_path, recalculate, do_listdir, + reclaim_age) + else: + return hashed, hashes diff --git a/swift/obj/mem_diskfile.py b/swift/obj/mem_diskfile.py index efb8c6c8c..be5fbf134 100644 --- a/swift/obj/mem_diskfile.py +++ b/swift/obj/mem_diskfile.py @@ -57,6 +57,12 @@ class InMemoryFileSystem(object): def get_diskfile(self, account, container, obj, **kwargs): return DiskFile(self, account, container, obj) + def pickle_async_update(self, *args, **kwargs): + """ + For now don't handle async updates. + """ + pass + class DiskFileWriter(object): """ @@ -98,6 +104,16 @@ class DiskFileWriter(object): metadata['name'] = self._name self._filesystem.put_object(self._name, self._fp, metadata) + def commit(self, timestamp): + """ + Perform any operations necessary to mark the object as durable. For + mem_diskfile type this is a no-op. + + :param timestamp: object put timestamp, an instance of + :class:`~swift.common.utils.Timestamp` + """ + pass + class DiskFileReader(object): """ diff --git a/swift/obj/mem_server.py b/swift/obj/mem_server.py index 83647661a..764a92a92 100644 --- a/swift/obj/mem_server.py +++ b/swift/obj/mem_server.py @@ -15,15 +15,7 @@ """ In-Memory Object Server for Swift """ -import os -from swift import gettext_ as _ -from eventlet import Timeout - -from swift.common.bufferedhttp import http_connect -from swift.common.exceptions import ConnectionTimeout - -from swift.common.http import is_success from swift.obj.mem_diskfile import InMemoryFileSystem from swift.obj import server @@ -53,49 +45,6 @@ class ObjectController(server.ObjectController): """ return self._filesystem.get_diskfile(account, container, obj, **kwargs) - def async_update(self, op, account, container, obj, host, partition, - contdevice, headers_out, objdevice, policy_idx): - """ - Sends or saves an async update. - - :param op: operation performed (ex: 'PUT', or 'DELETE') - :param account: account name for the object - :param container: container name for the object - :param obj: object name - :param host: host that the container is on - :param partition: partition that the container is on - :param contdevice: device name that the container is on - :param headers_out: dictionary of headers to send in the container - request - :param objdevice: device name that the object is in - :param policy_idx: the associated storage policy index - """ - headers_out['user-agent'] = 'object-server %s' % os.getpid() - full_path = '/%s/%s/%s' % (account, container, obj) - if all([host, partition, contdevice]): - try: - with ConnectionTimeout(self.conn_timeout): - ip, port = host.rsplit(':', 1) - conn = http_connect(ip, port, contdevice, partition, op, - full_path, headers_out) - with Timeout(self.node_timeout): - response = conn.getresponse() - response.read() - if is_success(response.status): - return - else: - self.logger.error(_( - 'ERROR Container update failed: %(status)d ' - 'response from %(ip)s:%(port)s/%(dev)s'), - {'status': response.status, 'ip': ip, 'port': port, - 'dev': contdevice}) - except (Exception, Timeout): - self.logger.exception(_( - 'ERROR container update failed with ' - '%(ip)s:%(port)s/%(dev)s'), - {'ip': ip, 'port': port, 'dev': contdevice}) - # FIXME: For now don't handle async updates - def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used diff --git a/swift/obj/reconstructor.py b/swift/obj/reconstructor.py new file mode 100644 index 000000000..0ee2afbf6 --- /dev/null +++ b/swift/obj/reconstructor.py @@ -0,0 +1,925 @@ +# Copyright (c) 2010-2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os.path import join +import random +import time +import itertools +from collections import defaultdict +import cPickle as pickle +import shutil + +from eventlet import (GreenPile, GreenPool, Timeout, sleep, hubs, tpool, + spawn) +from eventlet.support.greenlets import GreenletExit + +from swift import gettext_ as _ +from swift.common.utils import ( + whataremyips, unlink_older_than, compute_eta, get_logger, + dump_recon_cache, ismount, mkdirs, config_true_value, list_from_csv, + get_hub, tpool_reraise, GreenAsyncPile, Timestamp, remove_file) +from swift.common.swob import HeaderKeyDict +from swift.common.bufferedhttp import http_connect +from swift.common.daemon import Daemon +from swift.common.ring.utils import is_local_device +from swift.obj.ssync_sender import Sender as ssync_sender +from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE +from swift.obj.diskfile import DiskFileRouter, get_data_dir, \ + get_tmp_dir +from swift.common.storage_policy import POLICIES, EC_POLICY +from swift.common.exceptions import ConnectionTimeout, DiskFileError, \ + SuffixSyncError + +SYNC, REVERT = ('sync_only', 'sync_revert') + + +hubs.use_hub(get_hub()) + + +class RebuildingECDiskFileStream(object): + """ + This class wraps the the reconstructed fragment archive data and + metadata in the DiskFile interface for ssync. + """ + + def __init__(self, metadata, frag_index, rebuilt_fragment_iter): + # start with metadata from a participating FA + self.metadata = metadata + + # the new FA is going to have the same length as others in the set + self._content_length = self.metadata['Content-Length'] + + # update the FI and delete the ETag, the obj server will + # recalc on the other side... + self.metadata['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index + del self.metadata['ETag'] + + self.frag_index = frag_index + self.rebuilt_fragment_iter = rebuilt_fragment_iter + + def get_metadata(self): + return self.metadata + + @property + def content_length(self): + return self._content_length + + def reader(self): + for chunk in self.rebuilt_fragment_iter: + yield chunk + + +class ObjectReconstructor(Daemon): + """ + Reconstruct objects using erasure code. And also rebalance EC Fragment + Archive objects off handoff nodes. + + Encapsulates most logic and data needed by the object reconstruction + process. Each call to .reconstruct() performs one pass. It's up to the + caller to do this in a loop. + """ + + def __init__(self, conf, logger=None): + """ + :param conf: configuration object obtained from ConfigParser + :param logger: logging object + """ + self.conf = conf + self.logger = logger or get_logger( + conf, log_route='object-reconstructor') + self.devices_dir = conf.get('devices', '/srv/node') + self.mount_check = config_true_value(conf.get('mount_check', 'true')) + self.swift_dir = conf.get('swift_dir', '/etc/swift') + self.port = int(conf.get('bind_port', 6000)) + self.concurrency = int(conf.get('concurrency', 1)) + self.stats_interval = int(conf.get('stats_interval', '300')) + self.ring_check_interval = int(conf.get('ring_check_interval', 15)) + self.next_check = time.time() + self.ring_check_interval + self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) + self.partition_times = [] + self.run_pause = int(conf.get('run_pause', 30)) + self.http_timeout = int(conf.get('http_timeout', 60)) + self.lockup_timeout = int(conf.get('lockup_timeout', 1800)) + self.recon_cache_path = conf.get('recon_cache_path', + '/var/cache/swift') + self.rcache = os.path.join(self.recon_cache_path, "object.recon") + # defaults subject to change after beta + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.node_timeout = float(conf.get('node_timeout', 10)) + self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) + self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) + self.headers = { + 'Content-Length': '0', + 'user-agent': 'obj-reconstructor %s' % os.getpid()} + self.handoffs_first = config_true_value(conf.get('handoffs_first', + False)) + self._df_router = DiskFileRouter(conf, self.logger) + + def load_object_ring(self, policy): + """ + Make sure the policy's rings are loaded. + + :param policy: the StoragePolicy instance + :returns: appropriate ring object + """ + policy.load_ring(self.swift_dir) + return policy.object_ring + + def check_ring(self, object_ring): + """ + Check to see if the ring has been updated + + :param object_ring: the ring to check + :returns: boolean indicating whether or not the ring has changed + """ + if time.time() > self.next_check: + self.next_check = time.time() + self.ring_check_interval + if object_ring.has_changed(): + return False + return True + + def _full_path(self, node, part, path, policy): + return '%(replication_ip)s:%(replication_port)s' \ + '/%(device)s/%(part)s%(path)s ' \ + 'policy#%(policy)d frag#%(frag_index)s' % { + 'replication_ip': node['replication_ip'], + 'replication_port': node['replication_port'], + 'device': node['device'], + 'part': part, 'path': path, + 'policy': policy, + 'frag_index': node.get('index', 'handoff'), + } + + def _get_response(self, node, part, path, headers, policy): + """ + Helper method for reconstruction that GETs a single EC fragment + archive + + :param node: the node to GET from + :param part: the partition + :param path: full path of the desired EC archive + :param headers: the headers to send + :param policy: an instance of + :class:`~swift.common.storage_policy.BaseStoragePolicy` + :returns: response + """ + resp = None + headers['X-Backend-Node-Index'] = node['index'] + try: + with ConnectionTimeout(self.conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], + part, 'GET', path, headers=headers) + with Timeout(self.node_timeout): + resp = conn.getresponse() + if resp.status != HTTP_OK: + self.logger.warning( + _("Invalid response %(resp)s from %(full_path)s"), + {'resp': resp.status, + 'full_path': self._full_path(node, part, path, policy)}) + resp = None + except (Exception, Timeout): + self.logger.exception( + _("Trying to GET %(full_path)s"), { + 'full_path': self._full_path(node, part, path, policy)}) + return resp + + def reconstruct_fa(self, job, node, metadata): + """ + Reconstructs a fragment archive - this method is called from ssync + after a remote node responds that is missing this object - the local + diskfile is opened to provide metadata - but to reconstruct the + missing fragment archive we must connect to multiple object servers. + + :param job: job from ssync_sender + :param node: node that we're rebuilding to + :param metadata: the metadata to attach to the rebuilt archive + :returns: a DiskFile like class for use by ssync + :raises DiskFileError: if the fragment archive cannot be reconstructed + """ + + part_nodes = job['policy'].object_ring.get_part_nodes( + job['partition']) + part_nodes.remove(node) + + # the fragment index we need to reconstruct is the position index + # of the node we're rebuilding to within the primary part list + fi_to_rebuild = node['index'] + + # KISS send out connection requests to all nodes, see what sticks + headers = { + 'X-Backend-Storage-Policy-Index': int(job['policy']), + } + pile = GreenAsyncPile(len(part_nodes)) + path = metadata['name'] + for node in part_nodes: + pile.spawn(self._get_response, node, job['partition'], + path, headers, job['policy']) + responses = [] + etag = None + for resp in pile: + if not resp: + continue + resp.headers = HeaderKeyDict(resp.getheaders()) + responses.append(resp) + etag = sorted(responses, reverse=True, + key=lambda r: Timestamp( + r.headers.get('X-Backend-Timestamp') + ))[0].headers.get('X-Object-Sysmeta-Ec-Etag') + responses = [r for r in responses if + r.headers.get('X-Object-Sysmeta-Ec-Etag') == etag] + + if len(responses) >= job['policy'].ec_ndata: + break + else: + self.logger.error( + 'Unable to get enough responses (%s/%s) ' + 'to reconstruct %s with ETag %s' % ( + len(responses), job['policy'].ec_ndata, + self._full_path(node, job['partition'], + metadata['name'], job['policy']), + etag)) + raise DiskFileError('Unable to reconstruct EC archive') + + rebuilt_fragment_iter = self.make_rebuilt_fragment_iter( + responses[:job['policy'].ec_ndata], path, job['policy'], + fi_to_rebuild) + return RebuildingECDiskFileStream(metadata, fi_to_rebuild, + rebuilt_fragment_iter) + + def _reconstruct(self, policy, fragment_payload, frag_index): + # XXX with jerasure this doesn't work if we need to rebuild a + # parity fragment, and not all data fragments are available + # segment = policy.pyeclib_driver.reconstruct( + # fragment_payload, [frag_index])[0] + + # for safety until pyeclib 1.0.7 we'll just use decode and encode + segment = policy.pyeclib_driver.decode(fragment_payload) + return policy.pyeclib_driver.encode(segment)[frag_index] + + def make_rebuilt_fragment_iter(self, responses, path, policy, frag_index): + """ + Turn a set of connections from backend object servers into a generator + that yields up the rebuilt fragment archive for frag_index. + """ + + def _get_one_fragment(resp): + buff = '' + remaining_bytes = policy.fragment_size + while remaining_bytes: + chunk = resp.read(remaining_bytes) + if not chunk: + break + remaining_bytes -= len(chunk) + buff += chunk + return buff + + def fragment_payload_iter(): + # We need a fragment from each connections, so best to + # use a GreenPile to keep them ordered and in sync + pile = GreenPile(len(responses)) + while True: + for resp in responses: + pile.spawn(_get_one_fragment, resp) + try: + with Timeout(self.node_timeout): + fragment_payload = [fragment for fragment in pile] + except (Exception, Timeout): + self.logger.exception( + _("Error trying to rebuild %(path)s " + "policy#%(policy)d frag#%(frag_index)s"), { + 'path': path, + 'policy': policy, + 'frag_index': frag_index, + }) + break + if not all(fragment_payload): + break + rebuilt_fragment = self._reconstruct( + policy, fragment_payload, frag_index) + yield rebuilt_fragment + + return fragment_payload_iter() + + def stats_line(self): + """ + Logs various stats for the currently running reconstruction pass. + """ + if self.reconstruction_count: + elapsed = (time.time() - self.start) or 0.000001 + rate = self.reconstruction_count / elapsed + self.logger.info( + _("%(reconstructed)d/%(total)d (%(percentage).2f%%)" + " partitions reconstructed in %(time).2fs (%(rate).2f/sec, " + "%(remaining)s remaining)"), + {'reconstructed': self.reconstruction_count, + 'total': self.job_count, + 'percentage': + self.reconstruction_count * 100.0 / self.job_count, + 'time': time.time() - self.start, 'rate': rate, + 'remaining': '%d%s' % compute_eta(self.start, + self.reconstruction_count, + self.job_count)}) + if self.suffix_count: + self.logger.info( + _("%(checked)d suffixes checked - " + "%(hashed).2f%% hashed, %(synced).2f%% synced"), + {'checked': self.suffix_count, + 'hashed': (self.suffix_hash * 100.0) / self.suffix_count, + 'synced': (self.suffix_sync * 100.0) / self.suffix_count}) + self.partition_times.sort() + self.logger.info( + _("Partition times: max %(max).4fs, " + "min %(min).4fs, med %(med).4fs"), + {'max': self.partition_times[-1], + 'min': self.partition_times[0], + 'med': self.partition_times[ + len(self.partition_times) // 2]}) + else: + self.logger.info( + _("Nothing reconstructed for %s seconds."), + (time.time() - self.start)) + + def kill_coros(self): + """Utility function that kills all coroutines currently running.""" + for coro in list(self.run_pool.coroutines_running): + try: + coro.kill(GreenletExit) + except GreenletExit: + pass + + def heartbeat(self): + """ + Loop that runs in the background during reconstruction. It + periodically logs progress. + """ + while True: + sleep(self.stats_interval) + self.stats_line() + + def detect_lockups(self): + """ + In testing, the pool.waitall() call very occasionally failed to return. + This is an attempt to make sure the reconstructor finishes its + reconstruction pass in some eventuality. + """ + while True: + sleep(self.lockup_timeout) + if self.reconstruction_count == self.last_reconstruction_count: + self.logger.error(_("Lockup detected.. killing live coros.")) + self.kill_coros() + self.last_reconstruction_count = self.reconstruction_count + + def _get_partners(self, frag_index, part_nodes): + """ + Returns the left and right partners of the node whose index is + equal to the given frag_index. + + :param frag_index: a fragment index + :param part_nodes: a list of primary nodes + :returns: [<node-to-left>, <node-to-right>] + """ + return [ + part_nodes[(frag_index - 1) % len(part_nodes)], + part_nodes[(frag_index + 1) % len(part_nodes)], + ] + + def _get_hashes(self, policy, path, recalculate=None, do_listdir=False): + df_mgr = self._df_router[policy] + hashed, suffix_hashes = tpool_reraise( + df_mgr._get_hashes, path, recalculate=recalculate, + do_listdir=do_listdir, reclaim_age=self.reclaim_age) + self.logger.update_stats('suffix.hashes', hashed) + return suffix_hashes + + def get_suffix_delta(self, local_suff, local_index, + remote_suff, remote_index): + """ + Compare the local suffix hashes with the remote suffix hashes + for the given local and remote fragment indexes. Return those + suffixes which should be synced. + + :param local_suff: the local suffix hashes (from _get_hashes) + :param local_index: the local fragment index for the job + :param remote_suff: the remote suffix hashes (from remote + REPLICATE request) + :param remote_index: the remote fragment index for the job + + :returns: a list of strings, the suffix dirs to sync + """ + suffixes = [] + for suffix, sub_dict_local in local_suff.iteritems(): + sub_dict_remote = remote_suff.get(suffix, {}) + if (sub_dict_local.get(None) != sub_dict_remote.get(None) or + sub_dict_local.get(local_index) != + sub_dict_remote.get(remote_index)): + suffixes.append(suffix) + return suffixes + + def rehash_remote(self, node, job, suffixes): + try: + with Timeout(self.http_timeout): + conn = http_connect( + node['replication_ip'], node['replication_port'], + node['device'], job['partition'], 'REPLICATE', + '/' + '-'.join(sorted(suffixes)), + headers=self.headers) + conn.getresponse().read() + except (Exception, Timeout): + self.logger.exception( + _("Trying to sync suffixes with %s") % self._full_path( + node, job['partition'], '', job['policy'])) + + def _get_suffixes_to_sync(self, job, node): + """ + For SYNC jobs we need to make a remote REPLICATE request to get + the remote node's current suffix's hashes and then compare to our + local suffix's hashes to decide which suffixes (if any) are out + of sync. + + :param: the job dict, with the keys defined in ``_get_part_jobs`` + :param node: the remote node dict + :returns: a (possibly empty) list of strings, the suffixes to be + synced with the remote node. + """ + # get hashes from the remote node + remote_suffixes = None + try: + with Timeout(self.http_timeout): + resp = http_connect( + node['replication_ip'], node['replication_port'], + node['device'], job['partition'], 'REPLICATE', + '', headers=self.headers).getresponse() + if resp.status == HTTP_INSUFFICIENT_STORAGE: + self.logger.error( + _('%s responded as unmounted'), + self._full_path(node, job['partition'], '', + job['policy'])) + elif resp.status != HTTP_OK: + self.logger.error( + _("Invalid response %(resp)s " + "from %(full_path)s"), { + 'resp': resp.status, + 'full_path': self._full_path( + node, job['partition'], '', + job['policy']) + }) + else: + remote_suffixes = pickle.loads(resp.read()) + except (Exception, Timeout): + # all exceptions are logged here so that our caller can + # safely catch our exception and continue to the next node + # without logging + self.logger.exception('Unable to get remote suffix hashes ' + 'from %r' % self._full_path( + node, job['partition'], '', + job['policy'])) + + if remote_suffixes is None: + raise SuffixSyncError('Unable to get remote suffix hashes') + + suffixes = self.get_suffix_delta(job['hashes'], + job['frag_index'], + remote_suffixes, + node['index']) + # now recalculate local hashes for suffixes that don't + # match so we're comparing the latest + local_suff = self._get_hashes(job['policy'], job['path'], + recalculate=suffixes) + + suffixes = self.get_suffix_delta(local_suff, + job['frag_index'], + remote_suffixes, + node['index']) + + self.suffix_count += len(suffixes) + return suffixes + + def delete_reverted_objs(self, job, objects, frag_index): + """ + For EC we can potentially revert only some of a partition + so we'll delete reverted objects here. Note that we delete + the fragment index of the file we sent to the remote node. + + :param job: the job being processed + :param objects: a dict of objects to be deleted, each entry maps + hash=>timestamp + :param frag_index: (int) the fragment index of data files to be deleted + """ + df_mgr = self._df_router[job['policy']] + for object_hash, timestamp in objects.items(): + try: + df = df_mgr.get_diskfile_from_hash( + job['local_dev']['device'], job['partition'], + object_hash, job['policy'], + frag_index=frag_index) + df.purge(Timestamp(timestamp), frag_index) + except DiskFileError: + continue + + def process_job(self, job): + """ + Sync the local partition with the remote node(s) according to + the parameters of the job. For primary nodes, the SYNC job type + will define both left and right hand sync_to nodes to ssync with + as defined by this primary nodes index in the node list based on + the fragment index found in the partition. For non-primary + nodes (either handoff revert, or rebalance) the REVERT job will + define a single node in sync_to which is the proper/new home for + the fragment index. + + N.B. ring rebalancing can be time consuming and handoff nodes' + fragment indexes do not have a stable order, it's possible to + have more than one REVERT job for a partition, and in some rare + failure conditions there may even also be a SYNC job for the + same partition - but each one will be processed separately + because each job will define a separate list of node(s) to + 'sync_to'. + + :param: the job dict, with the keys defined in ``_get_job_info`` + """ + self.headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) + begin = time.time() + if job['job_type'] == REVERT: + self._revert(job, begin) + else: + self._sync(job, begin) + self.partition_times.append(time.time() - begin) + self.reconstruction_count += 1 + + def _sync(self, job, begin): + """ + Process a SYNC job. + """ + self.logger.increment( + 'partition.update.count.%s' % (job['local_dev']['device'],)) + # after our left and right partners, if there's some sort of + # failure we'll continue onto the remaining primary nodes and + # make sure they're in sync - or potentially rebuild missing + # fragments we find + dest_nodes = itertools.chain( + job['sync_to'], + # I think we could order these based on our index to better + # protect against a broken chain + itertools.ifilter( + lambda n: n['id'] not in (n['id'] for n in job['sync_to']), + job['policy'].object_ring.get_part_nodes(job['partition'])), + ) + syncd_with = 0 + for node in dest_nodes: + if syncd_with >= len(job['sync_to']): + # success! + break + + try: + suffixes = self._get_suffixes_to_sync(job, node) + except SuffixSyncError: + continue + + if not suffixes: + syncd_with += 1 + continue + + # ssync any out-of-sync suffixes with the remote node + success, _ = ssync_sender( + self, node, job, suffixes)() + # let remote end know to rehash it's suffixes + self.rehash_remote(node, job, suffixes) + # update stats for this attempt + self.suffix_sync += len(suffixes) + self.logger.update_stats('suffix.syncs', len(suffixes)) + if success: + syncd_with += 1 + self.logger.timing_since('partition.update.timing', begin) + + def _revert(self, job, begin): + """ + Process a REVERT job. + """ + self.logger.increment( + 'partition.delete.count.%s' % (job['local_dev']['device'],)) + # we'd desperately like to push this partition back to it's + # primary location, but if that node is down, the next best thing + # is one of the handoff locations - which *might* be us already! + dest_nodes = itertools.chain( + job['sync_to'], + job['policy'].object_ring.get_more_nodes(job['partition']), + ) + syncd_with = 0 + reverted_objs = {} + for node in dest_nodes: + if syncd_with >= len(job['sync_to']): + break + if node['id'] == job['local_dev']['id']: + # this is as good a place as any for this data for now + break + success, in_sync_objs = ssync_sender( + self, node, job, job['suffixes'])() + self.rehash_remote(node, job, job['suffixes']) + if success: + syncd_with += 1 + reverted_objs.update(in_sync_objs) + if syncd_with >= len(job['sync_to']): + self.delete_reverted_objs( + job, reverted_objs, job['frag_index']) + self.logger.timing_since('partition.delete.timing', begin) + + def _get_part_jobs(self, local_dev, part_path, partition, policy): + """ + Helper function to build jobs for a partition, this method will + read the suffix hashes and create job dictionaries to describe + the needed work. There will be one job for each fragment index + discovered in the partition. + + For a fragment index which corresponds to this node's ring + index, a job with job_type SYNC will be created to ensure that + the left and right hand primary ring nodes for the part have the + corresponding left and right hand fragment archives. + + A fragment index (or entire partition) for which this node is + not the primary corresponding node, will create job(s) with + job_type REVERT to ensure that fragment archives are pushed to + the correct node and removed from this one. + + A partition may result in multiple jobs. Potentially many + REVERT jobs, and zero or one SYNC job. + + :param local_dev: the local device + :param part_path: full path to partition + :param partition: partition number + :param policy: the policy + + :returns: a list of dicts of job info + """ + # find all the fi's in the part, and which suffixes have them + hashes = self._get_hashes(policy, part_path, do_listdir=True) + non_data_fragment_suffixes = [] + data_fi_to_suffixes = defaultdict(list) + for suffix, fi_hash in hashes.items(): + if not fi_hash: + # this is for sanity and clarity, normally an empty + # suffix would get del'd from the hashes dict, but an + # OSError trying to re-hash the suffix could leave the + # value empty - it will log the exception; but there's + # no way to properly address this suffix at this time. + continue + data_frag_indexes = [f for f in fi_hash if f is not None] + if not data_frag_indexes: + non_data_fragment_suffixes.append(suffix) + else: + for fi in data_frag_indexes: + data_fi_to_suffixes[fi].append(suffix) + + # helper to ensure consistent structure of jobs + def build_job(job_type, frag_index, suffixes, sync_to): + return { + 'job_type': job_type, + 'frag_index': frag_index, + 'suffixes': suffixes, + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': hashes, + 'policy': policy, + 'local_dev': local_dev, + # ssync likes to have it handy + 'device': local_dev['device'], + } + + # aggregate jobs for all the fragment index in this part + jobs = [] + + # check the primary nodes - to see if the part belongs here + part_nodes = policy.object_ring.get_part_nodes(partition) + for node in part_nodes: + if node['id'] == local_dev['id']: + # this partition belongs here, we'll need a sync job + frag_index = node['index'] + try: + suffixes = data_fi_to_suffixes.pop(frag_index) + except KeyError: + suffixes = [] + sync_job = build_job( + job_type=SYNC, + frag_index=frag_index, + suffixes=suffixes, + sync_to=self._get_partners(frag_index, part_nodes), + ) + # ssync callback to rebuild missing fragment_archives + sync_job['sync_diskfile_builder'] = self.reconstruct_fa + jobs.append(sync_job) + break + + # assign remaining data fragment suffixes to revert jobs + ordered_fis = sorted((len(suffixes), fi) for fi, suffixes + in data_fi_to_suffixes.items()) + for count, fi in ordered_fis: + revert_job = build_job( + job_type=REVERT, + frag_index=fi, + suffixes=data_fi_to_suffixes[fi], + sync_to=[part_nodes[fi]], + ) + jobs.append(revert_job) + + # now we need to assign suffixes that have no data fragments + if non_data_fragment_suffixes: + if jobs: + # the first job will be either the sync_job, or the + # revert_job for the fragment index that is most common + # among the suffixes + jobs[0]['suffixes'].extend(non_data_fragment_suffixes) + else: + # this is an unfortunate situation, we need a revert job to + # push partitions off this node, but none of the suffixes + # have any data fragments to hint at which node would be a + # good candidate to receive the tombstones. + jobs.append(build_job( + job_type=REVERT, + frag_index=None, + suffixes=non_data_fragment_suffixes, + # this is super safe + sync_to=part_nodes, + # something like this would be probably be better + # sync_to=random.sample(part_nodes, 3), + )) + # return a list of jobs for this part + return jobs + + def collect_parts(self, override_devices=None, + override_partitions=None): + """ + Helper for yielding partitions in the top level reconstructor + """ + override_devices = override_devices or [] + override_partitions = override_partitions or [] + ips = whataremyips() + for policy in POLICIES: + if policy.policy_type != EC_POLICY: + continue + self._diskfile_mgr = self._df_router[policy] + self.load_object_ring(policy) + data_dir = get_data_dir(policy) + local_devices = itertools.ifilter( + lambda dev: dev and is_local_device( + ips, self.port, + dev['replication_ip'], dev['replication_port']), + policy.object_ring.devs) + for local_dev in local_devices: + if override_devices and (local_dev['device'] not in + override_devices): + continue + dev_path = join(self.devices_dir, local_dev['device']) + obj_path = join(dev_path, data_dir) + tmp_path = join(dev_path, get_tmp_dir(int(policy))) + if self.mount_check and not ismount(dev_path): + self.logger.warn(_('%s is not mounted'), + local_dev['device']) + continue + unlink_older_than(tmp_path, time.time() - + self.reclaim_age) + if not os.path.exists(obj_path): + try: + mkdirs(obj_path) + except Exception: + self.logger.exception( + 'Unable to create %s' % obj_path) + continue + try: + partitions = os.listdir(obj_path) + except OSError: + self.logger.exception( + 'Unable to list partitions in %r' % obj_path) + continue + for partition in partitions: + part_path = join(obj_path, partition) + if not (partition.isdigit() and + os.path.isdir(part_path)): + self.logger.warning( + 'Unexpected entity in data dir: %r' % part_path) + remove_file(part_path) + continue + partition = int(partition) + if override_partitions and (partition not in + override_partitions): + continue + part_info = { + 'local_dev': local_dev, + 'policy': policy, + 'partition': partition, + 'part_path': part_path, + } + yield part_info + + def build_reconstruction_jobs(self, part_info): + """ + Helper function for collect_jobs to build jobs for reconstruction + using EC style storage policy + """ + jobs = self._get_part_jobs(**part_info) + random.shuffle(jobs) + if self.handoffs_first: + # Move the handoff revert jobs to the front of the list + jobs.sort(key=lambda job: job['job_type'], reverse=True) + self.job_count += len(jobs) + return jobs + + def _reset_stats(self): + self.start = time.time() + self.job_count = 0 + self.suffix_count = 0 + self.suffix_sync = 0 + self.suffix_hash = 0 + self.reconstruction_count = 0 + self.last_reconstruction_count = -1 + + def delete_partition(self, path): + self.logger.info(_("Removing partition: %s"), path) + tpool.execute(shutil.rmtree, path, ignore_errors=True) + + def reconstruct(self, **kwargs): + """Run a reconstruction pass""" + self._reset_stats() + self.partition_times = [] + + stats = spawn(self.heartbeat) + lockup_detector = spawn(self.detect_lockups) + sleep() # Give spawns a cycle + + try: + self.run_pool = GreenPool(size=self.concurrency) + for part_info in self.collect_parts(**kwargs): + if not self.check_ring(part_info['policy'].object_ring): + self.logger.info(_("Ring change detected. Aborting " + "current reconstruction pass.")) + return + jobs = self.build_reconstruction_jobs(part_info) + if not jobs: + # If this part belongs on this node, _get_part_jobs + # will *always* build a sync_job - even if there's + # no suffixes in the partition that needs to sync. + # If there's any suffixes in the partition then our + # job list would have *at least* one revert job. + # Therefore we know this part a) doesn't belong on + # this node and b) doesn't have any suffixes in it. + self.run_pool.spawn(self.delete_partition, + part_info['part_path']) + for job in jobs: + self.run_pool.spawn(self.process_job, job) + with Timeout(self.lockup_timeout): + self.run_pool.waitall() + except (Exception, Timeout): + self.logger.exception(_("Exception in top-level" + "reconstruction loop")) + self.kill_coros() + finally: + stats.kill() + lockup_detector.kill() + self.stats_line() + + def run_once(self, *args, **kwargs): + start = time.time() + self.logger.info(_("Running object reconstructor in script mode.")) + override_devices = list_from_csv(kwargs.get('devices')) + override_partitions = [int(p) for p in + list_from_csv(kwargs.get('partitions'))] + self.reconstruct( + override_devices=override_devices, + override_partitions=override_partitions) + total = (time.time() - start) / 60 + self.logger.info( + _("Object reconstruction complete (once). (%.02f minutes)"), total) + if not (override_partitions or override_devices): + dump_recon_cache({'object_reconstruction_time': total, + 'object_reconstruction_last': time.time()}, + self.rcache, self.logger) + + def run_forever(self, *args, **kwargs): + self.logger.info(_("Starting object reconstructor in daemon mode.")) + # Run the reconstructor continually + while True: + start = time.time() + self.logger.info(_("Starting object reconstruction pass.")) + # Run the reconstructor + self.reconstruct() + total = (time.time() - start) / 60 + self.logger.info( + _("Object reconstruction complete. (%.02f minutes)"), total) + dump_recon_cache({'object_reconstruction_time': total, + 'object_reconstruction_last': time.time()}, + self.rcache, self.logger) + self.logger.debug('reconstruction sleeping for %s seconds.', + self.run_pause) + sleep(self.run_pause) diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 5ee32884c..580d1827e 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -39,7 +39,7 @@ from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE from swift.obj import ssync_sender from swift.obj.diskfile import (DiskFileManager, get_hashes, get_data_dir, get_tmp_dir) -from swift.common.storage_policy import POLICIES +from swift.common.storage_policy import POLICIES, REPL_POLICY hubs.use_hub(get_hub()) @@ -110,14 +110,15 @@ class ObjectReplicator(Daemon): """ return self.sync_method(node, job, suffixes, *args, **kwargs) - def get_object_ring(self, policy_idx): + def load_object_ring(self, policy): """ - Get the ring object to use to handle a request based on its policy. + Make sure the policy's rings are loaded. - :policy_idx: policy index as defined in swift.conf + :param policy: the StoragePolicy instance :returns: appropriate ring object """ - return POLICIES.get_object_ring(policy_idx, self.swift_dir) + policy.load_ring(self.swift_dir) + return policy.object_ring def _rsync(self, args): """ @@ -170,7 +171,7 @@ class ObjectReplicator(Daemon): sync method in Swift. """ if not os.path.exists(job['path']): - return False, set() + return False, {} args = [ 'rsync', '--recursive', @@ -195,11 +196,11 @@ class ObjectReplicator(Daemon): args.append(spath) had_any = True if not had_any: - return False, set() - data_dir = get_data_dir(job['policy_idx']) + return False, {} + data_dir = get_data_dir(job['policy']) args.append(join(rsync_module, node['device'], data_dir, job['partition'])) - return self._rsync(args) == 0, set() + return self._rsync(args) == 0, {} def ssync(self, node, job, suffixes, remote_check_objs=None): return ssync_sender.Sender( @@ -231,7 +232,7 @@ class ObjectReplicator(Daemon): if len(suff) == 3 and isdir(join(path, suff))] self.replication_count += 1 self.logger.increment('partition.delete.count.%s' % (job['device'],)) - self.headers['X-Backend-Storage-Policy-Index'] = job['policy_idx'] + self.headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) begin = time.time() try: responses = [] @@ -245,8 +246,9 @@ class ObjectReplicator(Daemon): self.conf.get('sync_method', 'rsync') == 'ssync': kwargs['remote_check_objs'] = \ synced_remote_regions[node['region']] - # cand_objs is a list of objects for deletion - success, cand_objs = self.sync( + # candidates is a dict(hash=>timestamp) of objects + # for deletion + success, candidates = self.sync( node, job, suffixes, **kwargs) if success: with Timeout(self.http_timeout): @@ -257,7 +259,8 @@ class ObjectReplicator(Daemon): '/' + '-'.join(suffixes), headers=self.headers) conn.getresponse().read() if node['region'] != job['region']: - synced_remote_regions[node['region']] = cand_objs + synced_remote_regions[node['region']] = \ + candidates.keys() responses.append(success) for region, cand_objs in synced_remote_regions.iteritems(): if delete_objs is None: @@ -314,7 +317,7 @@ class ObjectReplicator(Daemon): """ self.replication_count += 1 self.logger.increment('partition.update.count.%s' % (job['device'],)) - self.headers['X-Backend-Storage-Policy-Index'] = job['policy_idx'] + self.headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) begin = time.time() try: hashed, local_hash = tpool_reraise( @@ -328,7 +331,8 @@ class ObjectReplicator(Daemon): random.shuffle(job['nodes']) nodes = itertools.chain( job['nodes'], - job['object_ring'].get_more_nodes(int(job['partition']))) + job['policy'].object_ring.get_more_nodes( + int(job['partition']))) while attempts_left > 0: # If this throws StopIteration it will be caught way below node = next(nodes) @@ -460,16 +464,15 @@ class ObjectReplicator(Daemon): self.kill_coros() self.last_replication_count = self.replication_count - def process_repl(self, policy, ips, override_devices=None, - override_partitions=None): + def build_replication_jobs(self, policy, ips, override_devices=None, + override_partitions=None): """ Helper function for collect_jobs to build jobs for replication using replication style storage policy """ jobs = [] - obj_ring = self.get_object_ring(policy.idx) - data_dir = get_data_dir(policy.idx) - for local_dev in [dev for dev in obj_ring.devs + data_dir = get_data_dir(policy) + for local_dev in [dev for dev in policy.object_ring.devs if (dev and is_local_device(ips, self.port, @@ -479,7 +482,7 @@ class ObjectReplicator(Daemon): or dev['device'] in override_devices))]: dev_path = join(self.devices_dir, local_dev['device']) obj_path = join(dev_path, data_dir) - tmp_path = join(dev_path, get_tmp_dir(int(policy))) + tmp_path = join(dev_path, get_tmp_dir(policy)) if self.mount_check and not ismount(dev_path): self.logger.warn(_('%s is not mounted'), local_dev['device']) continue @@ -497,7 +500,8 @@ class ObjectReplicator(Daemon): try: job_path = join(obj_path, partition) - part_nodes = obj_ring.get_part_nodes(int(partition)) + part_nodes = policy.object_ring.get_part_nodes( + int(partition)) nodes = [node for node in part_nodes if node['id'] != local_dev['id']] jobs.append( @@ -506,9 +510,8 @@ class ObjectReplicator(Daemon): obj_path=obj_path, nodes=nodes, delete=len(nodes) > len(part_nodes) - 1, - policy_idx=policy.idx, + policy=policy, partition=partition, - object_ring=obj_ring, region=local_dev['region'])) except ValueError: continue @@ -530,13 +533,15 @@ class ObjectReplicator(Daemon): jobs = [] ips = whataremyips() for policy in POLICIES: - if (override_policies is not None - and str(policy.idx) not in override_policies): - continue - # may need to branch here for future policy types - jobs += self.process_repl(policy, ips, - override_devices=override_devices, - override_partitions=override_partitions) + if policy.policy_type == REPL_POLICY: + if (override_policies is not None and + str(policy.idx) not in override_policies): + continue + # ensure rings are loaded for policy + self.load_object_ring(policy) + jobs += self.build_replication_jobs( + policy, ips, override_devices=override_devices, + override_partitions=override_partitions) random.shuffle(jobs) if self.handoffs_first: # Move the handoff parts to the front of the list @@ -569,7 +574,7 @@ class ObjectReplicator(Daemon): if self.mount_check and not ismount(dev_path): self.logger.warn(_('%s is not mounted'), job['device']) continue - if not self.check_ring(job['object_ring']): + if not self.check_ring(job['policy'].object_ring): self.logger.info(_("Ring change detected. Aborting " "current replication pass.")) return diff --git a/swift/obj/server.py b/swift/obj/server.py index ad0f9faeb..658f207a8 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -16,10 +16,12 @@ """ Object Server for Swift """ import cPickle as pickle +import json import os import multiprocessing import time import traceback +import rfc822 import socket import math from swift import gettext_ as _ @@ -30,7 +32,7 @@ from eventlet import sleep, wsgi, Timeout from swift.common.utils import public, get_logger, \ config_true_value, timing_stats, replication, \ normalize_delete_at_timestamp, get_log_line, Timestamp, \ - get_expirer_container + get_expirer_container, iter_multipart_mime_documents from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ valid_timestamp, check_utf8 @@ -48,8 +50,35 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \ HTTPInsufficientStorage, HTTPForbidden, HTTPException, HeaderKeyDict, \ - HTTPConflict -from swift.obj.diskfile import DATAFILE_SYSTEM_META, DiskFileManager + HTTPConflict, HTTPServerError +from swift.obj.diskfile import DATAFILE_SYSTEM_META, DiskFileRouter + + +def iter_mime_headers_and_bodies(wsgi_input, mime_boundary, read_chunk_size): + mime_documents_iter = iter_multipart_mime_documents( + wsgi_input, mime_boundary, read_chunk_size) + + for file_like in mime_documents_iter: + hdrs = HeaderKeyDict(rfc822.Message(file_like, 0)) + yield (hdrs, file_like) + + +def drain(file_like, read_size, timeout): + """ + Read and discard any bytes from file_like. + + :param file_like: file-like object to read from + :param read_size: how big a chunk to read at a time + :param timeout: how long to wait for a read (use None for no timeout) + + :raises ChunkReadTimeout: if no chunk was read in time + """ + + while True: + with ChunkReadTimeout(timeout): + chunk = file_like.read(read_size) + if not chunk: + break class EventletPlungerString(str): @@ -142,7 +171,7 @@ class ObjectController(BaseStorageServer): # Common on-disk hierarchy shared across account, container and object # servers. - self._diskfile_mgr = DiskFileManager(conf, self.logger) + self._diskfile_router = DiskFileRouter(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if 'replication_semaphore' in conf: @@ -156,7 +185,7 @@ class ObjectController(BaseStorageServer): conf.get('replication_failure_ratio') or 1.0) def get_diskfile(self, device, partition, account, container, obj, - policy_idx, **kwargs): + policy, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. @@ -165,11 +194,11 @@ class ObjectController(BaseStorageServer): DiskFile class would simply over-ride this method to provide that behavior. """ - return self._diskfile_mgr.get_diskfile( - device, partition, account, container, obj, policy_idx, **kwargs) + return self._diskfile_router[policy].get_diskfile( + device, partition, account, container, obj, policy, **kwargs) def async_update(self, op, account, container, obj, host, partition, - contdevice, headers_out, objdevice, policy_index): + contdevice, headers_out, objdevice, policy): """ Sends or saves an async update. @@ -183,7 +212,7 @@ class ObjectController(BaseStorageServer): :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in - :param policy_index: the associated storage policy index + :param policy: the associated BaseStoragePolicy instance """ headers_out['user-agent'] = 'object-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) @@ -213,12 +242,11 @@ class ObjectController(BaseStorageServer): data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} timestamp = headers_out['x-timestamp'] - self._diskfile_mgr.pickle_async_update(objdevice, account, container, - obj, data, timestamp, - policy_index) + self._diskfile_router[policy].pickle_async_update( + objdevice, account, container, obj, data, timestamp, policy) def container_update(self, op, account, container, obj, request, - headers_out, objdevice, policy_idx): + headers_out, objdevice, policy): """ Update the container when objects are updated. @@ -230,6 +258,7 @@ class ObjectController(BaseStorageServer): :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in + :param policy: the BaseStoragePolicy instance """ headers_in = request.headers conthosts = [h.strip() for h in @@ -255,14 +284,14 @@ class ObjectController(BaseStorageServer): headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() - headers_out['X-Backend-Storage-Policy-Index'] = policy_idx + headers_out['X-Backend-Storage-Policy-Index'] = int(policy) for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, - objdevice, policy_idx) + objdevice, policy) def delete_at_update(self, op, delete_at, account, container, obj, - request, objdevice, policy_index): + request, objdevice, policy): """ Update the expiring objects container when objects are updated. @@ -273,7 +302,7 @@ class ObjectController(BaseStorageServer): :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in - :param policy_index: the policy index to be used for tmp dir + :param policy: the BaseStoragePolicy instance (used for tmp dir) """ if config_true_value( request.headers.get('x-backend-replication', 'f')): @@ -333,13 +362,66 @@ class ObjectController(BaseStorageServer): op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice, - policy_index) + policy) + + def _make_timeout_reader(self, file_like): + def timeout_reader(): + with ChunkReadTimeout(self.client_timeout): + return file_like.read(self.network_chunk_size) + return timeout_reader + + def _read_put_commit_message(self, mime_documents_iter): + rcvd_commit = False + try: + with ChunkReadTimeout(self.client_timeout): + commit_hdrs, commit_iter = next(mime_documents_iter) + if commit_hdrs.get('X-Document', None) == "put commit": + rcvd_commit = True + drain(commit_iter, self.network_chunk_size, self.client_timeout) + except ChunkReadTimeout: + raise HTTPClientDisconnect() + except StopIteration: + raise HTTPBadRequest(body="couldn't find PUT commit MIME doc") + return rcvd_commit + + def _read_metadata_footer(self, mime_documents_iter): + try: + with ChunkReadTimeout(self.client_timeout): + footer_hdrs, footer_iter = next(mime_documents_iter) + except ChunkReadTimeout: + raise HTTPClientDisconnect() + except StopIteration: + raise HTTPBadRequest(body="couldn't find footer MIME doc") + + timeout_reader = self._make_timeout_reader(footer_iter) + try: + footer_body = ''.join(iter(timeout_reader, '')) + except ChunkReadTimeout: + raise HTTPClientDisconnect() + + footer_md5 = footer_hdrs.get('Content-MD5') + if not footer_md5: + raise HTTPBadRequest(body="no Content-MD5 in footer") + if footer_md5 != md5(footer_body).hexdigest(): + raise HTTPUnprocessableEntity(body="footer MD5 mismatch") + + try: + return HeaderKeyDict(json.loads(footer_body)) + except ValueError: + raise HTTPBadRequest("invalid JSON for footer doc") + + def _check_container_override(self, update_headers, metadata): + for key, val in metadata.iteritems(): + override_prefix = 'x-backend-container-update-override-' + if key.lower().startswith(override_prefix): + override = key.lower().replace(override_prefix, 'x-') + update_headers[override] = val @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" - device, partition, account, container, obj, policy_idx = \ + device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) @@ -349,7 +431,7 @@ class ObjectController(BaseStorageServer): try: disk_file = self.get_diskfile( device, partition, account, container, obj, - policy_idx=policy_idx) + policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: @@ -374,11 +456,11 @@ class ObjectController(BaseStorageServer): if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, - obj, request, device, policy_idx) + obj, request, device, policy) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, - policy_idx) + policy) try: disk_file.write_metadata(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): @@ -389,7 +471,7 @@ class ObjectController(BaseStorageServer): @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" - device, partition, account, container, obj, policy_idx = \ + device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) @@ -404,10 +486,22 @@ class ObjectController(BaseStorageServer): except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') + + # In case of multipart-MIME put, the proxy sends a chunked request, + # but may let us know the real content length so we can verify that + # we have enough disk space to hold the object. + if fsize is None: + fsize = request.headers.get('X-Backend-Obj-Content-Length') + if fsize is not None: + try: + fsize = int(fsize) + except ValueError as e: + return HTTPBadRequest(body=str(e), request=request, + content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj, - policy_idx=policy_idx) + policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: @@ -439,13 +533,51 @@ class ObjectController(BaseStorageServer): with disk_file.create(size=fsize) as writer: upload_size = 0 - def timeout_reader(): - with ChunkReadTimeout(self.client_timeout): - return request.environ['wsgi.input'].read( - self.network_chunk_size) + # If the proxy wants to send us object metadata after the + # object body, it sets some headers. We have to tell the + # proxy, in the 100 Continue response, that we're able to + # parse a multipart MIME document and extract the object and + # metadata from it. If we don't, then the proxy won't + # actually send the footer metadata. + have_metadata_footer = False + use_multiphase_commit = False + mime_documents_iter = iter([]) + obj_input = request.environ['wsgi.input'] + + hundred_continue_headers = [] + if config_true_value( + request.headers.get( + 'X-Backend-Obj-Multiphase-Commit')): + use_multiphase_commit = True + hundred_continue_headers.append( + ('X-Obj-Multiphase-Commit', 'yes')) + + if config_true_value( + request.headers.get('X-Backend-Obj-Metadata-Footer')): + have_metadata_footer = True + hundred_continue_headers.append( + ('X-Obj-Metadata-Footer', 'yes')) + + if have_metadata_footer or use_multiphase_commit: + obj_input.set_hundred_continue_response_headers( + hundred_continue_headers) + mime_boundary = request.headers.get( + 'X-Backend-Obj-Multipart-Mime-Boundary') + if not mime_boundary: + return HTTPBadRequest("no MIME boundary") + try: + with ChunkReadTimeout(self.client_timeout): + mime_documents_iter = iter_mime_headers_and_bodies( + request.environ['wsgi.input'], + mime_boundary, self.network_chunk_size) + _junk_hdrs, obj_input = next(mime_documents_iter) + except ChunkReadTimeout: + return HTTPRequestTimeout(request=request) + + timeout_reader = self._make_timeout_reader(obj_input) try: - for chunk in iter(lambda: timeout_reader(), ''): + for chunk in iter(timeout_reader, ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') @@ -461,9 +593,16 @@ class ObjectController(BaseStorageServer): upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) + + footer_meta = {} + if have_metadata_footer: + footer_meta = self._read_metadata_footer( + mime_documents_iter) + + request_etag = (footer_meta.get('etag') or + request.headers.get('etag', '')).lower() etag = etag.hexdigest() - if 'etag' in request.headers and \ - request.headers['etag'].lower() != etag: + if request_etag and request_etag != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.timestamp.internal, @@ -473,6 +612,8 @@ class ObjectController(BaseStorageServer): } metadata.update(val for val in request.headers.iteritems() if is_sys_or_user_meta('object', val[0])) + metadata.update(val for val in footer_meta.iteritems() + if is_sys_or_user_meta('object', val[0])) headers_to_copy = ( request.headers.get( 'X-Backend-Replication-Headers', '').split() + @@ -482,39 +623,63 @@ class ObjectController(BaseStorageServer): header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) + + # if the PUT requires a two-phase commit (a data and a commit + # phase) send the proxy server another 100-continue response + # to indicate that we are finished writing object data + if use_multiphase_commit: + request.environ['wsgi.input'].\ + send_hundred_continue_response() + if not self._read_put_commit_message(mime_documents_iter): + return HTTPServerError(request=request) + # got 2nd phase confirmation, write a timestamp.durable + # state file to indicate a successful PUT + + writer.commit(request.timestamp) + + # Drain any remaining MIME docs from the socket. There + # shouldn't be any, but we must read the whole request body. + try: + while True: + with ChunkReadTimeout(self.client_timeout): + _junk_hdrs, _junk_body = next(mime_documents_iter) + drain(_junk_body, self.network_chunk_size, + self.client_timeout) + except ChunkReadTimeout: + raise HTTPClientDisconnect() + except StopIteration: + pass + except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, - device, policy_idx) + device, policy) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, - request, device, policy_idx) + request, device, policy) update_headers = HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}) # apply any container update header overrides sent with request - for key, val in request.headers.iteritems(): - override_prefix = 'x-backend-container-update-override-' - if key.lower().startswith(override_prefix): - override = key.lower().replace(override_prefix, 'x-') - update_headers[override] = val + self._check_container_override(update_headers, request.headers) + self._check_container_override(update_headers, footer_meta) self.container_update( 'PUT', account, container, obj, request, update_headers, - device, policy_idx) + device, policy) return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" - device, partition, account, container, obj, policy_idx = \ + device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and @@ -522,7 +687,7 @@ class ObjectController(BaseStorageServer): try: disk_file = self.get_diskfile( device, partition, account, container, obj, - policy_idx=policy_idx) + policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: @@ -533,9 +698,14 @@ class ObjectController(BaseStorageServer): keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) + conditional_etag = None + if 'X-Backend-Etag-Is-At' in request.headers: + conditional_etag = metadata.get( + request.headers['X-Backend-Etag-Is-At']) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), - request=request, conditional_response=True) + request=request, conditional_response=True, + conditional_etag=conditional_etag) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): @@ -567,12 +737,12 @@ class ObjectController(BaseStorageServer): @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" - device, partition, account, container, obj, policy_idx = \ + device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) try: disk_file = self.get_diskfile( device, partition, account, container, obj, - policy_idx=policy_idx) + policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: @@ -585,7 +755,12 @@ class ObjectController(BaseStorageServer): headers['X-Backend-Timestamp'] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) - response = Response(request=request, conditional_response=True) + conditional_etag = None + if 'X-Backend-Etag-Is-At' in request.headers: + conditional_etag = metadata.get( + request.headers['X-Backend-Etag-Is-At']) + response = Response(request=request, conditional_response=True, + conditional_etag=conditional_etag) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): @@ -609,13 +784,13 @@ class ObjectController(BaseStorageServer): @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" - device, partition, account, container, obj, policy_idx = \ + device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) try: disk_file = self.get_diskfile( device, partition, account, container, obj, - policy_idx=policy_idx) + policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: @@ -667,13 +842,13 @@ class ObjectController(BaseStorageServer): if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, - policy_idx) + policy) if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, HeaderKeyDict({'x-timestamp': req_timestamp.internal}), - device, policy_idx) + device, policy) return response_class( request=request, headers={'X-Backend-Timestamp': response_timestamp.internal}) @@ -685,12 +860,17 @@ class ObjectController(BaseStorageServer): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. + + Note that the name REPLICATE is preserved for historical reasons as + this verb really just returns the hashes information for the specified + parameters and is used, for example, by both replication and EC. """ - device, partition, suffix, policy_idx = \ + device, partition, suffix_parts, policy = \ get_name_and_placement(request, 2, 3, True) + suffixes = suffix_parts.split('-') if suffix_parts else [] try: - hashes = self._diskfile_mgr.get_hashes(device, partition, suffix, - policy_idx) + hashes = self._diskfile_router[policy].get_hashes( + device, partition, suffixes, policy) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: @@ -700,7 +880,7 @@ class ObjectController(BaseStorageServer): @public @replication @timing_stats(sample_rate=0.1) - def REPLICATION(self, request): + def SSYNC(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): @@ -734,7 +914,7 @@ class ObjectController(BaseStorageServer): trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') - if req.method in ('REPLICATE', 'REPLICATION') or \ + if req.method in ('REPLICATE', 'SSYNC') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: diff --git a/swift/obj/ssync_receiver.py b/swift/obj/ssync_receiver.py index 248715d00..b636a1624 100644 --- a/swift/obj/ssync_receiver.py +++ b/swift/obj/ssync_receiver.py @@ -24,27 +24,28 @@ from swift.common import exceptions from swift.common import http from swift.common import swob from swift.common import utils +from swift.common import request_helpers class Receiver(object): """ - Handles incoming REPLICATION requests to the object server. + Handles incoming SSYNC requests to the object server. These requests come from the object-replicator daemon that uses :py:mod:`.ssync_sender`. - The number of concurrent REPLICATION requests is restricted by + The number of concurrent SSYNC requests is restricted by use of a replication_semaphore and can be configured with the object-server.conf [object-server] replication_concurrency setting. - A REPLICATION request is really just an HTTP conduit for + An SSYNC request is really just an HTTP conduit for sender/receiver replication communication. The overall - REPLICATION request should always succeed, but it will contain + SSYNC request should always succeed, but it will contain multiple requests within its request and response bodies. This "hack" is done so that replication concurrency can be managed. - The general process inside a REPLICATION request is: + The general process inside an SSYNC request is: 1. Initialize the request: Basic request validation, mount check, acquire semaphore lock, etc.. @@ -72,10 +73,10 @@ class Receiver(object): def __call__(self): """ - Processes a REPLICATION request. + Processes an SSYNC request. Acquires a semaphore lock and then proceeds through the steps - of the REPLICATION process. + of the SSYNC process. """ # The general theme for functions __call__ calls is that they should # raise exceptions.MessageTimeout for client timeouts (logged locally), @@ -88,7 +89,7 @@ class Receiver(object): try: # Double try blocks in case our main error handlers fail. try: - # intialize_request is for preamble items that can be done + # initialize_request is for preamble items that can be done # outside a replication semaphore lock. for data in self.initialize_request(): yield data @@ -98,7 +99,7 @@ class Receiver(object): if not self.app.replication_semaphore.acquire(False): raise swob.HTTPServiceUnavailable() try: - with self.app._diskfile_mgr.replication_lock(self.device): + with self.diskfile_mgr.replication_lock(self.device): for data in self.missing_check(): yield data for data in self.updates(): @@ -111,7 +112,7 @@ class Receiver(object): self.app.replication_semaphore.release() except exceptions.ReplicationLockTimeout as err: self.app.logger.debug( - '%s/%s/%s REPLICATION LOCK TIMEOUT: %s' % ( + '%s/%s/%s SSYNC LOCK TIMEOUT: %s' % ( self.request.remote_addr, self.device, self.partition, err)) yield ':ERROR: %d %r\n' % (0, str(err)) @@ -166,14 +167,17 @@ class Receiver(object): """ # The following is the setting we talk about above in _ensure_flush. self.request.environ['eventlet.minimum_write_chunk_size'] = 0 - self.device, self.partition = utils.split_path( - urllib.unquote(self.request.path), 2, 2, False) - self.policy_idx = \ - int(self.request.headers.get('X-Backend-Storage-Policy-Index', 0)) + self.device, self.partition, self.policy = \ + request_helpers.get_name_and_placement(self.request, 2, 2, False) + if 'X-Backend-Ssync-Frag-Index' in self.request.headers: + self.frag_index = int( + self.request.headers['X-Backend-Ssync-Frag-Index']) + else: + self.frag_index = None utils.validate_device_partition(self.device, self.partition) - if self.app._diskfile_mgr.mount_check and \ - not constraints.check_mount( - self.app._diskfile_mgr.devices, self.device): + self.diskfile_mgr = self.app._diskfile_router[self.policy] + if self.diskfile_mgr.mount_check and not constraints.check_mount( + self.diskfile_mgr.devices, self.device): raise swob.HTTPInsufficientStorage(drive=self.device) self.fp = self.request.environ['wsgi.input'] for data in self._ensure_flush(): @@ -182,7 +186,7 @@ class Receiver(object): def missing_check(self): """ Handles the receiver-side of the MISSING_CHECK step of a - REPLICATION request. + SSYNC request. Receives a list of hashes and timestamps of object information the sender can provide and responds with a list @@ -226,11 +230,13 @@ class Receiver(object): line = self.fp.readline(self.app.network_chunk_size) if not line or line.strip() == ':MISSING_CHECK: END': break - object_hash, timestamp = [urllib.unquote(v) for v in line.split()] + parts = line.split() + object_hash, timestamp = [urllib.unquote(v) for v in parts[:2]] want = False try: - df = self.app._diskfile_mgr.get_diskfile_from_hash( - self.device, self.partition, object_hash, self.policy_idx) + df = self.diskfile_mgr.get_diskfile_from_hash( + self.device, self.partition, object_hash, self.policy, + frag_index=self.frag_index) except exceptions.DiskFileNotExist: want = True else: @@ -253,7 +259,7 @@ class Receiver(object): def updates(self): """ - Handles the UPDATES step of a REPLICATION request. + Handles the UPDATES step of an SSYNC request. Receives a set of PUT and DELETE subrequests that will be routed to the object server itself for processing. These @@ -353,7 +359,7 @@ class Receiver(object): subreq_iter()) else: raise Exception('Invalid subrequest method %s' % method) - subreq.headers['X-Backend-Storage-Policy-Index'] = self.policy_idx + subreq.headers['X-Backend-Storage-Policy-Index'] = int(self.policy) subreq.headers['X-Backend-Replication'] = 'True' if replication_headers: subreq.headers['X-Backend-Replication-Headers'] = \ diff --git a/swift/obj/ssync_sender.py b/swift/obj/ssync_sender.py index 1058ab262..8e9202c00 100644 --- a/swift/obj/ssync_sender.py +++ b/swift/obj/ssync_sender.py @@ -22,7 +22,7 @@ from swift.common import http class Sender(object): """ - Sends REPLICATION requests to the object server. + Sends SSYNC requests to the object server. These requests are eventually handled by :py:mod:`.ssync_receiver` and full documentation about the @@ -31,6 +31,7 @@ class Sender(object): def __init__(self, daemon, node, job, suffixes, remote_check_objs=None): self.daemon = daemon + self.df_mgr = self.daemon._diskfile_mgr self.node = node self.job = job self.suffixes = suffixes @@ -38,28 +39,28 @@ class Sender(object): self.response = None self.response_buffer = '' self.response_chunk_left = 0 - self.available_set = set() + # available_map has an entry for each object in given suffixes that + # is available to be sync'd; each entry is a hash => timestamp + self.available_map = {} # When remote_check_objs is given in job, ssync_sender trys only to # make sure those objects exist or not in remote. self.remote_check_objs = remote_check_objs + # send_list has an entry for each object that the receiver wants to + # be sync'ed; each entry is an object hash self.send_list = [] self.failures = 0 - @property - def policy_idx(self): - return int(self.job.get('policy_idx', 0)) - def __call__(self): """ Perform ssync with remote node. - :returns: a 2-tuple, in the form (success, can_delete_objs). - - Success is a boolean, and can_delete_objs is an iterable of strings - representing the hashes which are in sync with the remote node. + :returns: a 2-tuple, in the form (success, can_delete_objs) where + success is a boolean and can_delete_objs is the map of + objects that are in sync with the receiver. Each entry in + can_delete_objs maps a hash => timestamp """ if not self.suffixes: - return True, set() + return True, {} try: # Double try blocks in case our main error handler fails. try: @@ -72,18 +73,20 @@ class Sender(object): self.missing_check() if self.remote_check_objs is None: self.updates() - can_delete_obj = self.available_set + can_delete_obj = self.available_map else: # when we are initialized with remote_check_objs we don't # *send* any requested updates; instead we only collect # what's already in sync and safe for deletion - can_delete_obj = self.available_set.difference( - self.send_list) + in_sync_hashes = (set(self.available_map.keys()) - + set(self.send_list)) + can_delete_obj = dict((hash_, self.available_map[hash_]) + for hash_ in in_sync_hashes) self.disconnect() if not self.failures: return True, can_delete_obj else: - return False, set() + return False, {} except (exceptions.MessageTimeout, exceptions.ReplicationException) as err: self.daemon.logger.error( @@ -109,11 +112,11 @@ class Sender(object): # would only get called if the above except Exception handler # failed (bad node or job data). self.daemon.logger.exception('EXCEPTION in replication.Sender') - return False, set() + return False, {} def connect(self): """ - Establishes a connection and starts a REPLICATION request + Establishes a connection and starts an SSYNC request with the object server. """ with exceptions.MessageTimeout( @@ -121,11 +124,13 @@ class Sender(object): self.connection = bufferedhttp.BufferedHTTPConnection( '%s:%s' % (self.node['replication_ip'], self.node['replication_port'])) - self.connection.putrequest('REPLICATION', '/%s/%s' % ( + self.connection.putrequest('SSYNC', '/%s/%s' % ( self.node['device'], self.job['partition'])) self.connection.putheader('Transfer-Encoding', 'chunked') self.connection.putheader('X-Backend-Storage-Policy-Index', - self.policy_idx) + int(self.job['policy'])) + self.connection.putheader('X-Backend-Ssync-Frag-Index', + self.node['index']) self.connection.endheaders() with exceptions.MessageTimeout( self.daemon.node_timeout, 'connect receive'): @@ -137,7 +142,7 @@ class Sender(object): def readline(self): """ - Reads a line from the REPLICATION response body. + Reads a line from the SSYNC response body. httplib has no readline and will block on read(x) until x is read, so we have to do the work ourselves. A bit of this is @@ -183,7 +188,7 @@ class Sender(object): def missing_check(self): """ Handles the sender-side of the MISSING_CHECK step of a - REPLICATION request. + SSYNC request. Full documentation of this can be found at :py:meth:`.Receiver.missing_check`. @@ -193,14 +198,15 @@ class Sender(object): self.daemon.node_timeout, 'missing_check start'): msg = ':MISSING_CHECK: START\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) - hash_gen = self.daemon._diskfile_mgr.yield_hashes( + hash_gen = self.df_mgr.yield_hashes( self.job['device'], self.job['partition'], - self.policy_idx, self.suffixes) + self.job['policy'], self.suffixes, + frag_index=self.job.get('frag_index')) if self.remote_check_objs is not None: hash_gen = ifilter(lambda (path, object_hash, timestamp): object_hash in self.remote_check_objs, hash_gen) for path, object_hash, timestamp in hash_gen: - self.available_set.add(object_hash) + self.available_map[object_hash] = timestamp with exceptions.MessageTimeout( self.daemon.node_timeout, 'missing_check send line'): @@ -234,12 +240,13 @@ class Sender(object): line = line.strip() if line == ':MISSING_CHECK: END': break - if line: - self.send_list.append(line) + parts = line.split() + if parts: + self.send_list.append(parts[0]) def updates(self): """ - Handles the sender-side of the UPDATES step of a REPLICATION + Handles the sender-side of the UPDATES step of an SSYNC request. Full documentation of this can be found at @@ -252,15 +259,19 @@ class Sender(object): self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) for object_hash in self.send_list: try: - df = self.daemon._diskfile_mgr.get_diskfile_from_hash( + df = self.df_mgr.get_diskfile_from_hash( self.job['device'], self.job['partition'], object_hash, - self.policy_idx) + self.job['policy'], frag_index=self.job.get('frag_index')) except exceptions.DiskFileNotExist: continue url_path = urllib.quote( '/%s/%s/%s' % (df.account, df.container, df.obj)) try: df.open() + # EC reconstructor may have passed a callback to build + # an alternative diskfile... + df = self.job.get('sync_diskfile_builder', lambda *args: df)( + self.job, self.node, df.get_metadata()) except exceptions.DiskFileDeleted as err: self.send_delete(url_path, err.timestamp) except exceptions.DiskFileError: @@ -328,7 +339,7 @@ class Sender(object): def disconnect(self): """ Closes down the connection to the object server once done - with the REPLICATION request. + with the SSYNC request. """ try: with exceptions.MessageTimeout( diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 6c40c456a..f5d1f37fa 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -29,7 +29,8 @@ from swift.common.ring import Ring from swift.common.utils import get_logger, renamer, write_pickle, \ dump_recon_cache, config_true_value, ismount from swift.common.daemon import Daemon -from swift.obj.diskfile import get_tmp_dir, get_async_dir, ASYNCDIR_BASE +from swift.common.storage_policy import split_policy_string, PolicyError +from swift.obj.diskfile import get_tmp_dir, ASYNCDIR_BASE from swift.common.http import is_success, HTTP_NOT_FOUND, \ HTTP_INTERNAL_SERVER_ERROR @@ -148,28 +149,19 @@ class ObjectUpdater(Daemon): start_time = time.time() # loop through async pending dirs for all policies for asyncdir in self._listdir(device): - # skip stuff like "accounts", "containers", etc. - if not (asyncdir == ASYNCDIR_BASE or - asyncdir.startswith(ASYNCDIR_BASE + '-')): - continue - # we only care about directories async_pending = os.path.join(device, asyncdir) if not os.path.isdir(async_pending): continue - - if asyncdir == ASYNCDIR_BASE: - policy_idx = 0 - else: - _junk, policy_idx = asyncdir.split('-', 1) - try: - policy_idx = int(policy_idx) - get_async_dir(policy_idx) - except ValueError: - self.logger.warn(_('Directory %s does not map to a ' - 'valid policy') % asyncdir) - continue - + if not asyncdir.startswith(ASYNCDIR_BASE): + # skip stuff like "accounts", "containers", etc. + continue + try: + base, policy = split_policy_string(asyncdir) + except PolicyError as e: + self.logger.warn(_('Directory %r does not map ' + 'to a valid policy (%s)') % (asyncdir, e)) + continue for prefix in self._listdir(async_pending): prefix_path = os.path.join(async_pending, prefix) if not os.path.isdir(prefix_path): @@ -193,7 +185,7 @@ class ObjectUpdater(Daemon): os.unlink(update_path) else: self.process_object_update(update_path, device, - policy_idx) + policy) last_obj_hash = obj_hash time.sleep(self.slowdown) try: @@ -202,13 +194,13 @@ class ObjectUpdater(Daemon): pass self.logger.timing_since('timing', start_time) - def process_object_update(self, update_path, device, policy_idx): + def process_object_update(self, update_path, device, policy): """ Process the object information to be updated and update. :param update_path: path to pickled object update file :param device: path to device - :param policy_idx: storage policy index of object update + :param policy: storage policy of object update """ try: update = pickle.load(open(update_path, 'rb')) @@ -228,7 +220,7 @@ class ObjectUpdater(Daemon): headers_out = update['headers'].copy() headers_out['user-agent'] = 'object-updater %s' % os.getpid() headers_out.setdefault('X-Backend-Storage-Policy-Index', - str(policy_idx)) + str(int(policy))) events = [spawn(self.object_update, node, part, update['op'], obj, headers_out) for node in nodes if node['id'] not in successes] @@ -256,7 +248,7 @@ class ObjectUpdater(Daemon): if new_successes: update['successes'] = successes write_pickle(update, update_path, os.path.join( - device, get_tmp_dir(policy_idx))) + device, get_tmp_dir(policy))) def object_update(self, node, part, op, obj, headers_out): """ diff --git a/swift/proxy/controllers/__init__.py b/swift/proxy/controllers/__init__.py index de4c0145b..706fd9165 100644 --- a/swift/proxy/controllers/__init__.py +++ b/swift/proxy/controllers/__init__.py @@ -13,7 +13,7 @@ from swift.proxy.controllers.base import Controller from swift.proxy.controllers.info import InfoController -from swift.proxy.controllers.obj import ObjectController +from swift.proxy.controllers.obj import ObjectControllerRouter from swift.proxy.controllers.account import AccountController from swift.proxy.controllers.container import ContainerController @@ -22,5 +22,5 @@ __all__ = [ 'ContainerController', 'Controller', 'InfoController', - 'ObjectController', + 'ObjectControllerRouter', ] diff --git a/swift/proxy/controllers/account.py b/swift/proxy/controllers/account.py index ea2f8ae33..915e1c481 100644 --- a/swift/proxy/controllers/account.py +++ b/swift/proxy/controllers/account.py @@ -58,9 +58,10 @@ class AccountController(Controller): constraints.MAX_ACCOUNT_NAME_LENGTH) return resp - partition, nodes = self.app.account_ring.get_nodes(self.account_name) + partition = self.app.account_ring.get_part(self.account_name) + node_iter = self.app.iter_nodes(self.app.account_ring, partition) resp = self.GETorHEAD_base( - req, _('Account'), self.app.account_ring, partition, + req, _('Account'), node_iter, partition, req.swift_entity_path.rstrip('/')) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get('X-Account-Status', '').lower() == 'deleted': diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 0aeb803f1..ca12d343e 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -28,6 +28,7 @@ import os import time import functools import inspect +import logging import operator from sys import exc_info from swift import gettext_ as _ @@ -39,14 +40,14 @@ from eventlet.timeout import Timeout from swift.common.wsgi import make_pre_authed_env from swift.common.utils import Timestamp, config_true_value, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \ - quorum_size, GreenAsyncPile + GreenAsyncPile, quorum_size, parse_content_range from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \ ConnectionTimeout from swift.common.http import is_informational, is_success, is_redirection, \ is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \ HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \ - HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED + HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE from swift.common.swob import Request, Response, HeaderKeyDict, Range, \ HTTPException, HTTPRequestedRangeNotSatisfiable from swift.common.request_helpers import strip_sys_meta_prefix, \ @@ -593,16 +594,37 @@ def close_swift_conn(src): pass +def bytes_to_skip(record_size, range_start): + """ + Assume an object is composed of N records, where the first N-1 are all + the same size and the last is at most that large, but may be smaller. + + When a range request is made, it might start with a partial record. This + must be discarded, lest the consumer get bad data. This is particularly + true of suffix-byte-range requests, e.g. "Range: bytes=-12345" where the + size of the object is unknown at the time the request is made. + + This function computes the number of bytes that must be discarded to + ensure only whole records are yielded. Erasure-code decoding needs this. + + This function could have been inlined, but it took enough tries to get + right that some targeted unit tests were desirable, hence its extraction. + """ + return (record_size - (range_start % record_size)) % record_size + + class GetOrHeadHandler(object): - def __init__(self, app, req, server_type, ring, partition, path, - backend_headers): + def __init__(self, app, req, server_type, node_iter, partition, path, + backend_headers, client_chunk_size=None): self.app = app - self.ring = ring + self.node_iter = node_iter self.server_type = server_type self.partition = partition self.path = path self.backend_headers = backend_headers + self.client_chunk_size = client_chunk_size + self.skip_bytes = 0 self.used_nodes = [] self.used_source_etag = '' @@ -649,6 +671,35 @@ class GetOrHeadHandler(object): else: self.backend_headers['Range'] = 'bytes=%d-' % num_bytes + def learn_size_from_content_range(self, start, end): + """ + If client_chunk_size is set, makes sure we yield things starting on + chunk boundaries based on the Content-Range header in the response. + + Sets our first Range header to the value learned from the + Content-Range header in the response; if we were given a + fully-specified range (e.g. "bytes=123-456"), this is a no-op. + + If we were given a half-specified range (e.g. "bytes=123-" or + "bytes=-456"), then this changes the Range header to a + semantically-equivalent one *and* it lets us resume on a proper + boundary instead of just in the middle of a piece somewhere. + + If the original request is for more than one range, this does not + affect our backend Range header, since we don't support resuming one + of those anyway. + """ + if self.client_chunk_size: + self.skip_bytes = bytes_to_skip(self.client_chunk_size, start) + + if 'Range' in self.backend_headers: + req_range = Range(self.backend_headers['Range']) + + if len(req_range.ranges) > 1: + return + + self.backend_headers['Range'] = "bytes=%d-%d" % (start, end) + def is_good_source(self, src): """ Indicates whether or not the request made to the backend found @@ -674,42 +725,74 @@ class GetOrHeadHandler(object): """ try: nchunks = 0 - bytes_read_from_source = 0 + client_chunk_size = self.client_chunk_size + bytes_consumed_from_backend = 0 node_timeout = self.app.node_timeout if self.server_type == 'Object': node_timeout = self.app.recoverable_node_timeout + buf = '' while True: try: with ChunkReadTimeout(node_timeout): chunk = source.read(self.app.object_chunk_size) nchunks += 1 - bytes_read_from_source += len(chunk) + buf += chunk except ChunkReadTimeout: exc_type, exc_value, exc_traceback = exc_info() if self.newest or self.server_type != 'Object': raise exc_type, exc_value, exc_traceback try: - self.fast_forward(bytes_read_from_source) + self.fast_forward(bytes_consumed_from_backend) except (NotImplementedError, HTTPException, ValueError): raise exc_type, exc_value, exc_traceback + buf = '' new_source, new_node = self._get_source_and_node() if new_source: self.app.exception_occurred( node, _('Object'), - _('Trying to read during GET (retrying)')) + _('Trying to read during GET (retrying)'), + level=logging.ERROR, exc_info=( + exc_type, exc_value, exc_traceback)) # Close-out the connection as best as possible. if getattr(source, 'swift_conn', None): close_swift_conn(source) source = new_source node = new_node - bytes_read_from_source = 0 continue else: raise exc_type, exc_value, exc_traceback + + if buf and self.skip_bytes: + if self.skip_bytes < len(buf): + buf = buf[self.skip_bytes:] + bytes_consumed_from_backend += self.skip_bytes + self.skip_bytes = 0 + else: + self.skip_bytes -= len(buf) + bytes_consumed_from_backend += len(buf) + buf = '' + if not chunk: + if buf: + with ChunkWriteTimeout(self.app.client_timeout): + bytes_consumed_from_backend += len(buf) + yield buf + buf = '' break - with ChunkWriteTimeout(self.app.client_timeout): - yield chunk + + if client_chunk_size is not None: + while len(buf) >= client_chunk_size: + client_chunk = buf[:client_chunk_size] + buf = buf[client_chunk_size:] + with ChunkWriteTimeout(self.app.client_timeout): + yield client_chunk + bytes_consumed_from_backend += len(client_chunk) + else: + with ChunkWriteTimeout(self.app.client_timeout): + yield buf + bytes_consumed_from_backend += len(buf) + buf = '' + # This is for fairness; if the network is outpacing the CPU, # we'll always be able to read and write data without # encountering an EWOULDBLOCK, and so eventlet will not switch @@ -757,7 +840,7 @@ class GetOrHeadHandler(object): node_timeout = self.app.node_timeout if self.server_type == 'Object' and not self.newest: node_timeout = self.app.recoverable_node_timeout - for node in self.app.iter_nodes(self.ring, self.partition): + for node in self.node_iter: if node in self.used_nodes: continue start_node_timing = time.time() @@ -793,8 +876,10 @@ class GetOrHeadHandler(object): src_headers = dict( (k.lower(), v) for k, v in possible_source.getheaders()) - if src_headers.get('etag', '').strip('"') != \ - self.used_source_etag: + + if self.used_source_etag != src_headers.get( + 'x-object-sysmeta-ec-etag', + src_headers.get('etag', '')).strip('"'): self.statuses.append(HTTP_NOT_FOUND) self.reasons.append('') self.bodies.append('') @@ -832,7 +917,9 @@ class GetOrHeadHandler(object): src_headers = dict( (k.lower(), v) for k, v in possible_source.getheaders()) - self.used_source_etag = src_headers.get('etag', '').strip('"') + self.used_source_etag = src_headers.get( + 'x-object-sysmeta-ec-etag', + src_headers.get('etag', '')).strip('"') return source, node return None, None @@ -841,13 +928,17 @@ class GetOrHeadHandler(object): res = None if source: res = Response(request=req) + res.status = source.status + update_headers(res, source.getheaders()) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): + cr = res.headers.get('Content-Range') + if cr: + start, end, total = parse_content_range(cr) + self.learn_size_from_content_range(start, end) res.app_iter = self._make_app_iter(req, node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn - res.status = source.status - update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ @@ -993,7 +1084,8 @@ class Controller(object): else: info['partition'] = part info['nodes'] = nodes - info.setdefault('storage_policy', '0') + if info.get('storage_policy') is None: + info['storage_policy'] = 0 return info def _make_request(self, nodes, part, method, path, headers, query, @@ -1098,6 +1190,13 @@ class Controller(object): '%s %s' % (self.server_type, req.method), overrides=overrides, headers=resp_headers) + def _quorum_size(self, n): + """ + Number of successful backend responses needed for the proxy to + consider the client request successful. + """ + return quorum_size(n) + def have_quorum(self, statuses, node_count): """ Given a list of statuses from several requests, determine if @@ -1107,16 +1206,18 @@ class Controller(object): :param node_count: number of nodes being queried (basically ring count) :returns: True or False, depending on if quorum is established """ - quorum = quorum_size(node_count) + quorum = self._quorum_size(node_count) if len(statuses) >= quorum: - for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): + for hundred in (HTTP_CONTINUE, HTTP_OK, HTTP_MULTIPLE_CHOICES, + HTTP_BAD_REQUEST): if sum(1 for s in statuses if hundred <= s < hundred + 100) >= quorum: return True return False def best_response(self, req, statuses, reasons, bodies, server_type, - etag=None, headers=None, overrides=None): + etag=None, headers=None, overrides=None, + quorum_size=None): """ Given a list of responses from several servers, choose the best to return to the API. @@ -1128,10 +1229,16 @@ class Controller(object): :param server_type: type of server the responses came from :param etag: etag :param headers: headers of each response + :param overrides: overrides to apply when lacking quorum + :param quorum_size: quorum size to use :returns: swob.Response object with the correct status, body, etc. set """ + if quorum_size is None: + quorum_size = self._quorum_size(len(statuses)) + resp = self._compute_quorum_response( - req, statuses, reasons, bodies, etag, headers) + req, statuses, reasons, bodies, etag, headers, + quorum_size=quorum_size) if overrides and not resp: faked_up_status_indices = set() transformed = [] @@ -1145,7 +1252,8 @@ class Controller(object): statuses, reasons, headers, bodies = zip(*transformed) resp = self._compute_quorum_response( req, statuses, reasons, bodies, etag, headers, - indices_to_avoid=faked_up_status_indices) + indices_to_avoid=faked_up_status_indices, + quorum_size=quorum_size) if not resp: resp = Response(request=req) @@ -1156,14 +1264,14 @@ class Controller(object): return resp def _compute_quorum_response(self, req, statuses, reasons, bodies, etag, - headers, indices_to_avoid=()): + headers, quorum_size, indices_to_avoid=()): if not statuses: return None for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = \ [(i, s) for i, s in enumerate(statuses) if hundred <= s < hundred + 100] - if len(hstatuses) >= quorum_size(len(statuses)): + if len(hstatuses) >= quorum_size: resp = Response(request=req) try: status_index, status = max( @@ -1228,22 +1336,25 @@ class Controller(object): else: self.app.logger.warning('Could not autocreate account %r' % path) - def GETorHEAD_base(self, req, server_type, ring, partition, path): + def GETorHEAD_base(self, req, server_type, node_iter, partition, path, + client_chunk_size=None): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type used in logging - :param ring: the ring to obtain nodes from + :param node_iter: an iterator to obtain nodes from :param partition: partition :param path: path for the request + :param client_chunk_size: chunk size for response body iterator :returns: swob.Response object """ backend_headers = self.generate_request_headers( req, additional=req.headers) - handler = GetOrHeadHandler(self.app, req, self.server_type, ring, - partition, path, backend_headers) + handler = GetOrHeadHandler(self.app, req, self.server_type, node_iter, + partition, path, backend_headers, + client_chunk_size=client_chunk_size) res = handler.get_working_response(req) if not res: diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index fb422e68d..3e4a2bb03 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -93,8 +93,9 @@ class ContainerController(Controller): return HTTPNotFound(request=req) part = self.app.container_ring.get_part( self.account_name, self.container_name) + node_iter = self.app.iter_nodes(self.app.container_ring, part) resp = self.GETorHEAD_base( - req, _('Container'), self.app.container_ring, part, + req, _('Container'), node_iter, part, req.swift_entity_path) if 'swift.authorize' in req.environ: req.acl = resp.headers.get('x-container-read') diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 2b53ba7a8..a83242b5f 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -24,13 +24,17 @@ # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. +import collections import itertools import mimetypes import time import math +import random +from hashlib import md5 from swift import gettext_ as _ from urllib import unquote, quote +from greenlet import GreenletExit from eventlet import GreenPile from eventlet.queue import Queue from eventlet.timeout import Timeout @@ -38,7 +42,8 @@ from eventlet.timeout import Timeout from swift.common.utils import ( clean_content_type, config_true_value, ContextPool, csv_append, GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp, - normalize_delete_at_timestamp, public, quorum_size, get_expirer_container) + normalize_delete_at_timestamp, public, get_expirer_container, + quorum_size) from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_copy_from_header, check_destination_header, \ @@ -46,21 +51,24 @@ from swift.common.constraints import check_metadata, check_object_creation, \ from swift.common import constraints from swift.common.exceptions import ChunkReadTimeout, \ ChunkWriteTimeout, ConnectionTimeout, ListingIterNotFound, \ - ListingIterNotAuthorized, ListingIterError + ListingIterNotAuthorized, ListingIterError, ResponseTimeout, \ + InsufficientStorage, FooterNotSupported, MultiphasePUTNotSupported, \ + PutterConnectError from swift.common.http import ( is_success, is_client_error, is_server_error, HTTP_CONTINUE, HTTP_CREATED, HTTP_MULTIPLE_CHOICES, HTTP_NOT_FOUND, HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, HTTP_INSUFFICIENT_STORAGE, - HTTP_PRECONDITION_FAILED, HTTP_CONFLICT) -from swift.common.storage_policy import POLICIES + HTTP_PRECONDITION_FAILED, HTTP_CONFLICT, is_informational) +from swift.common.storage_policy import (POLICIES, REPL_POLICY, EC_POLICY, + ECDriverError, PolicyError) from swift.proxy.controllers.base import Controller, delay_denial, \ cors_validation from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ - HTTPServerError, HTTPServiceUnavailable, Request, \ - HTTPClientDisconnect, HeaderKeyDict, HTTPException + HTTPServerError, HTTPServiceUnavailable, Request, HeaderKeyDict, \ + HTTPClientDisconnect, HTTPUnprocessableEntity, Response, HTTPException from swift.common.request_helpers import is_sys_or_user_meta, is_sys_meta, \ - remove_items, copy_header_subset + remove_items, copy_header_subset, close_if_possible def copy_headers_into(from_r, to_r): @@ -85,8 +93,41 @@ def check_content_type(req): return None -class ObjectController(Controller): - """WSGI controller for object requests.""" +class ObjectControllerRouter(object): + + policy_type_to_controller_map = {} + + @classmethod + def register(cls, policy_type): + """ + Decorator for Storage Policy implemenations to register + their ObjectController implementations. + + This also fills in a policy_type attribute on the class. + """ + def register_wrapper(controller_cls): + if policy_type in cls.policy_type_to_controller_map: + raise PolicyError( + '%r is already registered for the policy_type %r' % ( + cls.policy_type_to_controller_map[policy_type], + policy_type)) + cls.policy_type_to_controller_map[policy_type] = controller_cls + controller_cls.policy_type = policy_type + return controller_cls + return register_wrapper + + def __init__(self): + self.policy_to_controller_cls = {} + for policy in POLICIES: + self.policy_to_controller_cls[policy] = \ + self.policy_type_to_controller_map[policy.policy_type] + + def __getitem__(self, policy): + return self.policy_to_controller_cls[policy] + + +class BaseObjectController(Controller): + """Base WSGI controller for object requests.""" server_type = 'Object' def __init__(self, app, account_name, container_name, object_name, @@ -114,8 +155,10 @@ class ObjectController(Controller): lreq.environ['QUERY_STRING'] = \ 'format=json&prefix=%s&marker=%s' % (quote(lprefix), quote(marker)) + container_node_iter = self.app.iter_nodes(self.app.container_ring, + lpartition) lresp = self.GETorHEAD_base( - lreq, _('Container'), self.app.container_ring, lpartition, + lreq, _('Container'), container_node_iter, lpartition, lreq.swift_entity_path) if 'swift.authorize' in env: lreq.acl = lresp.headers.get('x-container-read') @@ -180,6 +223,7 @@ class ObjectController(Controller): # pass the policy index to storage nodes via req header policy_index = req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy']) + policy = POLICIES.get_by_index(policy_index) obj_ring = self.app.get_object_ring(policy_index) req.headers['X-Backend-Storage-Policy-Index'] = policy_index if 'swift.authorize' in req.environ: @@ -188,9 +232,10 @@ class ObjectController(Controller): return aresp partition = obj_ring.get_part( self.account_name, self.container_name, self.object_name) - resp = self.GETorHEAD_base( - req, _('Object'), obj_ring, partition, - req.swift_entity_path) + node_iter = self.app.iter_nodes(obj_ring, partition) + + resp = self._reroute(policy)._get_or_head_response( + req, node_iter, partition, policy) if ';' in resp.headers.get('content-type', ''): resp.content_type = clean_content_type( @@ -383,7 +428,10 @@ class ObjectController(Controller): _('Trying to get final status of PUT to %s') % req.path) return (None, None) - def _get_put_responses(self, req, conns, nodes): + def _get_put_responses(self, req, conns, nodes, **kwargs): + """ + Collect replicated object responses. + """ statuses = [] reasons = [] bodies = [] @@ -488,6 +536,7 @@ class ObjectController(Controller): self.object_name = src_obj_name self.container_name = src_container_name self.account_name = src_account_name + source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, @@ -589,8 +638,9 @@ class ObjectController(Controller): 'X-Newest': 'True'} hreq = Request.blank(req.path_info, headers=_headers, environ={'REQUEST_METHOD': 'HEAD'}) + hnode_iter = self.app.iter_nodes(obj_ring, partition) hresp = self.GETorHEAD_base( - hreq, _('Object'), obj_ring, partition, + hreq, _('Object'), hnode_iter, partition, hreq.swift_entity_path) is_manifest = 'X-Object-Manifest' in req.headers or \ @@ -654,7 +704,10 @@ class ObjectController(Controller): req.headers['X-Timestamp'] = Timestamp(time.time()).internal return None - def _check_failure_put_connections(self, conns, req, nodes): + def _check_failure_put_connections(self, conns, req, nodes, min_conns): + """ + Identify any failed connections and check minimum connection count. + """ if req.if_none_match is not None and '*' in req.if_none_match: statuses = [conn.resp.status for conn in conns if conn.resp] if HTTP_PRECONDITION_FAILED in statuses: @@ -675,7 +728,6 @@ class ObjectController(Controller): 'timestamps': ', '.join(timestamps)}) raise HTTPAccepted(request=req) - min_conns = quorum_size(len(nodes)) self._check_min_conn(req, conns, min_conns) def _get_put_connections(self, req, nodes, partition, outgoing_headers, @@ -709,8 +761,12 @@ class ObjectController(Controller): raise HTTPServiceUnavailable(request=req) def _transfer_data(self, req, data_source, conns, nodes): - min_conns = quorum_size(len(nodes)) + """ + Transfer data for a replicated object. + This method was added in the PUT method extraction change + """ + min_conns = quorum_size(len(nodes)) bytes_transferred = 0 try: with ContextPool(len(nodes)) as pool: @@ -775,11 +831,11 @@ class ObjectController(Controller): This method is responsible for establishing connection with storage nodes and sending object to each one of those - nodes. After sending the data, the "best" reponse will be + nodes. After sending the data, the "best" response will be returned based on statuses from all connections """ - policy_idx = req.headers.get('X-Backend-Storage-Policy-Index') - policy = POLICIES.get_by_index(policy_idx) + policy_index = req.headers.get('X-Backend-Storage-Policy-Index') + policy = POLICIES.get_by_index(policy_index) if not nodes: return HTTPNotFound() @@ -790,11 +846,11 @@ class ObjectController(Controller): expect = False conns = self._get_put_connections(req, nodes, partition, outgoing_headers, policy, expect) - + min_conns = quorum_size(len(nodes)) try: # check that a minimum number of connections were established and # meet all the correct conditions set in the request - self._check_failure_put_connections(conns, req, nodes) + self._check_failure_put_connections(conns, req, nodes, min_conns) # transfer data self._transfer_data(req, data_source, conns, nodes) @@ -1015,6 +1071,21 @@ class ObjectController(Controller): headers, overrides=status_overrides) return resp + def _reroute(self, policy): + """ + For COPY requests we need to make sure the controller instance the + request is routed through is the correct type for the policy. + """ + if not policy: + raise HTTPServiceUnavailable('Unknown Storage Policy') + if policy.policy_type != self.policy_type: + controller = self.app.obj_controller_router[policy]( + self.app, self.account_name, self.container_name, + self.object_name) + else: + controller = self + return controller + @public @cors_validation @delay_denial @@ -1031,6 +1102,7 @@ class ObjectController(Controller): self.account_name = dest_account del req.headers['Destination-Account'] dest_container, dest_object = check_destination_header(req) + source = '/%s/%s' % (self.container_name, self.object_name) self.container_name = dest_container self.object_name = dest_object @@ -1042,4 +1114,1109 @@ class ObjectController(Controller): req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote(source) del req.headers['Destination'] - return self.PUT(req) + + container_info = self.container_info( + dest_account, dest_container, req) + dest_policy = POLICIES.get_by_index(container_info['storage_policy']) + + return self._reroute(dest_policy).PUT(req) + + +@ObjectControllerRouter.register(REPL_POLICY) +class ReplicatedObjectController(BaseObjectController): + + def _get_or_head_response(self, req, node_iter, partition, policy): + resp = self.GETorHEAD_base( + req, _('Object'), node_iter, partition, + req.swift_entity_path) + return resp + + +class ECAppIter(object): + """ + WSGI iterable that decodes EC fragment archives (or portions thereof) + into the original object (or portions thereof). + + :param path: path for the request + + :param policy: storage policy for this object + + :param internal_app_iters: list of the WSGI iterables from object server + GET responses for fragment archives. For an M+K erasure code, the + caller must supply M such iterables. + + :param range_specs: list of dictionaries describing the ranges requested + by the client. Each dictionary contains the start and end of the + client's requested byte range as well as the start and end of the EC + segments containing that byte range. + + :param obj_length: length of the object, in bytes. Learned from the + headers in the GET response from the object server. + + :param logger: a logger + """ + def __init__(self, path, policy, internal_app_iters, range_specs, + obj_length, logger): + self.path = path + self.policy = policy + self.internal_app_iters = internal_app_iters + self.range_specs = range_specs + self.obj_length = obj_length + self.boundary = '' + self.logger = logger + + def close(self): + for it in self.internal_app_iters: + close_if_possible(it) + + def __iter__(self): + segments_iter = self.decode_segments_from_fragments() + + if len(self.range_specs) == 0: + # plain GET; just yield up segments + for seg in segments_iter: + yield seg + return + + if len(self.range_specs) > 1: + raise NotImplementedError("multi-range GETs not done yet") + + for range_spec in self.range_specs: + client_start = range_spec['client_start'] + client_end = range_spec['client_end'] + segment_start = range_spec['segment_start'] + segment_end = range_spec['segment_end'] + + seg_size = self.policy.ec_segment_size + is_suffix = client_start is None + + if is_suffix: + # Suffix byte ranges (i.e. requests for the last N bytes of + # an object) are likely to end up not on a segment boundary. + client_range_len = client_end + client_start = max(self.obj_length - client_range_len, 0) + client_end = self.obj_length - 1 + + # may be mid-segment; if it is, then everything up to the + # first segment boundary is garbage, and is discarded before + # ever getting into this function. + unaligned_segment_start = max(self.obj_length - segment_end, 0) + alignment_offset = ( + (seg_size - (unaligned_segment_start % seg_size)) + % seg_size) + segment_start = unaligned_segment_start + alignment_offset + segment_end = self.obj_length - 1 + else: + # It's entirely possible that the client asked for a range that + # includes some bytes we have and some we don't; for example, a + # range of bytes 1000-20000000 on a 1500-byte object. + segment_end = (min(segment_end, self.obj_length - 1) + if segment_end is not None + else self.obj_length - 1) + client_end = (min(client_end, self.obj_length - 1) + if client_end is not None + else self.obj_length - 1) + + num_segments = int( + math.ceil(float(segment_end + 1 - segment_start) + / self.policy.ec_segment_size)) + # We get full segments here, but the client may have requested a + # byte range that begins or ends in the middle of a segment. + # Thus, we have some amount of overrun (extra decoded bytes) + # that we trim off so the client gets exactly what they + # requested. + start_overrun = client_start - segment_start + end_overrun = segment_end - client_end + + for i, next_seg in enumerate(segments_iter): + # We may have a start_overrun of more than one segment in + # the case of suffix-byte-range requests. However, we never + # have an end_overrun of more than one segment. + if start_overrun > 0: + seglen = len(next_seg) + if seglen <= start_overrun: + start_overrun -= seglen + continue + else: + next_seg = next_seg[start_overrun:] + start_overrun = 0 + + if i == (num_segments - 1) and end_overrun: + next_seg = next_seg[:-end_overrun] + + yield next_seg + + def decode_segments_from_fragments(self): + # Decodes the fragments from the object servers and yields one + # segment at a time. + queues = [Queue(1) for _junk in range(len(self.internal_app_iters))] + + def put_fragments_in_queue(frag_iter, queue): + try: + for fragment in frag_iter: + if fragment[0] == ' ': + raise Exception('Leading whitespace on fragment.') + queue.put(fragment) + except GreenletExit: + # killed by contextpool + pass + except ChunkReadTimeout: + # unable to resume in GetOrHeadHandler + pass + except: # noqa + self.logger.exception("Exception fetching fragments for %r" % + self.path) + finally: + queue.resize(2) # ensure there's room + queue.put(None) + + with ContextPool(len(self.internal_app_iters)) as pool: + for app_iter, queue in zip( + self.internal_app_iters, queues): + pool.spawn(put_fragments_in_queue, app_iter, queue) + + while True: + fragments = [] + for qi, queue in enumerate(queues): + fragment = queue.get() + queue.task_done() + fragments.append(fragment) + + # If any object server connection yields out a None; we're + # done. Either they are all None, and we've finished + # successfully; or some un-recoverable failure has left us + # with an un-reconstructible list of fragments - so we'll + # break out of the iter so WSGI can tear down the broken + # connection. + if not all(fragments): + break + try: + segment = self.policy.pyeclib_driver.decode(fragments) + except ECDriverError: + self.logger.exception("Error decoding fragments for %r" % + self.path) + raise + + yield segment + + def app_iter_range(self, start, end): + return self + + def app_iter_ranges(self, content_type, boundary, content_size): + self.boundary = boundary + + +def client_range_to_segment_range(client_start, client_end, segment_size): + """ + Takes a byterange from the client and converts it into a byterange + spanning the necessary segments. + + Handles prefix, suffix, and fully-specified byte ranges. + + Examples: + client_range_to_segment_range(100, 700, 512) = (0, 1023) + client_range_to_segment_range(100, 700, 256) = (0, 767) + client_range_to_segment_range(300, None, 256) = (256, None) + + :param client_start: first byte of the range requested by the client + :param client_end: last byte of the range requested by the client + :param segment_size: size of an EC segment, in bytes + + :returns: a 2-tuple (seg_start, seg_end) where + + * seg_start is the first byte of the first segment, or None if this is + a suffix byte range + + * seg_end is the last byte of the last segment, or None if this is a + prefix byte range + """ + # the index of the first byte of the first segment + segment_start = ( + int(client_start // segment_size) + * segment_size) if client_start is not None else None + # the index of the last byte of the last segment + segment_end = ( + # bytes M- + None if client_end is None else + # bytes M-N + (((int(client_end // segment_size) + 1) + * segment_size) - 1) if client_start is not None else + # bytes -N: we get some extra bytes to make sure we + # have all we need. + # + # To see why, imagine a 100-byte segment size, a + # 340-byte object, and a request for the last 50 + # bytes. Naively requesting the last 100 bytes would + # result in a truncated first segment and hence a + # truncated download. (Of course, the actual + # obj-server requests are for fragments, not + # segments, but that doesn't change the + # calculation.) + # + # This does mean that we fetch an extra segment if + # the object size is an exact multiple of the + # segment size. It's a little wasteful, but it's + # better to be a little wasteful than to get some + # range requests completely wrong. + (int(math.ceil(( + float(client_end) / segment_size) + 1)) # nsegs + * segment_size)) + return (segment_start, segment_end) + + +def segment_range_to_fragment_range(segment_start, segment_end, segment_size, + fragment_size): + """ + Takes a byterange spanning some segments and converts that into a + byterange spanning the corresponding fragments within their fragment + archives. + + Handles prefix, suffix, and fully-specified byte ranges. + + :param segment_start: first byte of the first segment + :param segment_end: last byte of the last segment + :param segment_size: size of an EC segment, in bytes + :param fragment_size: size of an EC fragment, in bytes + + :returns: a 2-tuple (frag_start, frag_end) where + + * frag_start is the first byte of the first fragment, or None if this + is a suffix byte range + + * frag_end is the last byte of the last fragment, or None if this is a + prefix byte range + """ + # Note: segment_start and (segment_end + 1) are + # multiples of segment_size, so we don't have to worry + # about integer math giving us rounding troubles. + # + # There's a whole bunch of +1 and -1 in here; that's because HTTP wants + # byteranges to be inclusive of the start and end, so e.g. bytes 200-300 + # is a range containing 101 bytes. Python has half-inclusive ranges, of + # course, so we have to convert back and forth. We try to keep things in + # HTTP-style byteranges for consistency. + + # the index of the first byte of the first fragment + fragment_start = (( + segment_start / segment_size * fragment_size) + if segment_start is not None else None) + # the index of the last byte of the last fragment + fragment_end = ( + # range unbounded on the right + None if segment_end is None else + # range unbounded on the left; no -1 since we're + # asking for the last N bytes, not to have a + # particular byte be the last one + ((segment_end + 1) / segment_size + * fragment_size) if segment_start is None else + # range bounded on both sides; the -1 is because the + # rest of the expression computes the length of the + # fragment, and a range of N bytes starts at index M + # and ends at M + N - 1. + ((segment_end + 1) / segment_size * fragment_size) - 1) + return (fragment_start, fragment_end) + + +NO_DATA_SENT = 1 +SENDING_DATA = 2 +DATA_SENT = 3 +DATA_ACKED = 4 +COMMIT_SENT = 5 + + +class ECPutter(object): + """ + This is here mostly to wrap up the fact that all EC PUTs are + chunked because of the mime boundary footer trick and the first + half of the two-phase PUT conversation handling. + + An HTTP PUT request that supports streaming. + + Probably deserves more docs than this, but meh. + """ + def __init__(self, conn, node, resp, path, connect_duration, + mime_boundary): + # Note: you probably want to call Putter.connect() instead of + # instantiating one of these directly. + self.conn = conn + self.node = node + self.resp = resp + self.path = path + self.connect_duration = connect_duration + # for handoff nodes node_index is None + self.node_index = node.get('index') + self.mime_boundary = mime_boundary + self.chunk_hasher = md5() + + self.failed = False + self.queue = None + self.state = NO_DATA_SENT + + def current_status(self): + """ + Returns the current status of the response. + + A response starts off with no current status, then may or may not have + a status of 100 for some time, and then ultimately has a final status + like 200, 404, et cetera. + """ + return self.resp.status + + def await_response(self, timeout, informational=False): + """ + Get 100-continue response indicating the end of 1st phase of a 2-phase + commit or the final response, i.e. the one with status >= 200. + + Might or might not actually wait for anything. If we said Expect: + 100-continue but got back a non-100 response, that'll be the thing + returned, and we won't do any network IO to get it. OTOH, if we got + a 100 Continue response and sent up the PUT request's body, then + we'll actually read the 2xx-5xx response off the network here. + + :returns: HTTPResponse + :raises: Timeout if the response took too long + """ + conn = self.conn + with Timeout(timeout): + if not conn.resp: + if informational: + self.resp = conn.getexpect() + else: + self.resp = conn.getresponse() + return self.resp + + def spawn_sender_greenthread(self, pool, queue_depth, write_timeout, + exception_handler): + """Call before sending the first chunk of request body""" + self.queue = Queue(queue_depth) + pool.spawn(self._send_file, write_timeout, exception_handler) + + def wait(self): + if self.queue.unfinished_tasks: + self.queue.join() + + def _start_mime_doc_object_body(self): + self.queue.put("--%s\r\nX-Document: object body\r\n\r\n" % + (self.mime_boundary,)) + + def send_chunk(self, chunk): + if not chunk: + # If we're not using chunked transfer-encoding, sending a 0-byte + # chunk is just wasteful. If we *are* using chunked + # transfer-encoding, sending a 0-byte chunk terminates the + # request body. Neither one of these is good. + return + elif self.state == DATA_SENT: + raise ValueError("called send_chunk after end_of_object_data") + + if self.state == NO_DATA_SENT and self.mime_boundary: + # We're sending the object plus other stuff in the same request + # body, all wrapped up in multipart MIME, so we'd better start + # off the MIME document before sending any object data. + self._start_mime_doc_object_body() + self.state = SENDING_DATA + + self.queue.put(chunk) + + def end_of_object_data(self, footer_metadata): + """ + Call when there is no more data to send. + + :param footer_metadata: dictionary of metadata items + """ + if self.state == DATA_SENT: + raise ValueError("called end_of_object_data twice") + elif self.state == NO_DATA_SENT and self.mime_boundary: + self._start_mime_doc_object_body() + + footer_body = json.dumps(footer_metadata) + footer_md5 = md5(footer_body).hexdigest() + + tail_boundary = ("--%s" % (self.mime_boundary,)) + + message_parts = [ + ("\r\n--%s\r\n" % self.mime_boundary), + "X-Document: object metadata\r\n", + "Content-MD5: %s\r\n" % footer_md5, + "\r\n", + footer_body, "\r\n", + tail_boundary, "\r\n", + ] + self.queue.put("".join(message_parts)) + + self.queue.put('') + self.state = DATA_SENT + + def send_commit_confirmation(self): + """ + Call when there are > quorum 2XX responses received. Send commit + confirmations to all object nodes to finalize the PUT. + """ + if self.state == COMMIT_SENT: + raise ValueError("called send_commit_confirmation twice") + + self.state = DATA_ACKED + + if self.mime_boundary: + body = "put_commit_confirmation" + tail_boundary = ("--%s--" % (self.mime_boundary,)) + message_parts = [ + "X-Document: put commit\r\n", + "\r\n", + body, "\r\n", + tail_boundary, + ] + self.queue.put("".join(message_parts)) + + self.queue.put('') + self.state = COMMIT_SENT + + def _send_file(self, write_timeout, exception_handler): + """ + Method for a file PUT coro. Takes chunks from a queue and sends them + down a socket. + + If something goes wrong, the "failed" attribute will be set to true + and the exception handler will be called. + """ + while True: + chunk = self.queue.get() + if not self.failed: + to_send = "%x\r\n%s\r\n" % (len(chunk), chunk) + try: + with ChunkWriteTimeout(write_timeout): + self.conn.send(to_send) + except (Exception, ChunkWriteTimeout): + self.failed = True + exception_handler(self.conn.node, _('Object'), + _('Trying to write to %s') % self.path) + self.queue.task_done() + + @classmethod + def connect(cls, node, part, path, headers, conn_timeout, node_timeout, + chunked=False): + """ + Connect to a backend node and send the headers. + + :returns: Putter instance + + :raises: ConnectionTimeout if initial connection timed out + :raises: ResponseTimeout if header retrieval timed out + :raises: InsufficientStorage on 507 response from node + :raises: PutterConnectError on non-507 server error response from node + :raises: FooterNotSupported if need_metadata_footer is set but + backend node can't process footers + :raises: MultiphasePUTNotSupported if need_multiphase_support is + set but backend node can't handle multiphase PUT + """ + mime_boundary = "%.64x" % random.randint(0, 16 ** 64) + headers = HeaderKeyDict(headers) + # We're going to be adding some unknown amount of data to the + # request, so we can't use an explicit content length, and thus + # we must use chunked encoding. + headers['Transfer-Encoding'] = 'chunked' + headers['Expect'] = '100-continue' + if 'Content-Length' in headers: + headers['X-Backend-Obj-Content-Length'] = \ + headers.pop('Content-Length') + + headers['X-Backend-Obj-Multipart-Mime-Boundary'] = mime_boundary + + headers['X-Backend-Obj-Metadata-Footer'] = 'yes' + + headers['X-Backend-Obj-Multiphase-Commit'] = 'yes' + + start_time = time.time() + with ConnectionTimeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], + part, 'PUT', path, headers) + connect_duration = time.time() - start_time + + with ResponseTimeout(node_timeout): + resp = conn.getexpect() + + if resp.status == HTTP_INSUFFICIENT_STORAGE: + raise InsufficientStorage + + if is_server_error(resp.status): + raise PutterConnectError(resp.status) + + if is_informational(resp.status): + continue_headers = HeaderKeyDict(resp.getheaders()) + can_send_metadata_footer = config_true_value( + continue_headers.get('X-Obj-Metadata-Footer', 'no')) + can_handle_multiphase_put = config_true_value( + continue_headers.get('X-Obj-Multiphase-Commit', 'no')) + + if not can_send_metadata_footer: + raise FooterNotSupported() + + if not can_handle_multiphase_put: + raise MultiphasePUTNotSupported() + + conn.node = node + conn.resp = None + if is_success(resp.status) or resp.status == HTTP_CONFLICT: + conn.resp = resp + elif (headers.get('If-None-Match', None) is not None and + resp.status == HTTP_PRECONDITION_FAILED): + conn.resp = resp + + return cls(conn, node, resp, path, connect_duration, mime_boundary) + + +def chunk_transformer(policy, nstreams): + segment_size = policy.ec_segment_size + + buf = collections.deque() + total_buf_len = 0 + + chunk = yield + while chunk: + buf.append(chunk) + total_buf_len += len(chunk) + if total_buf_len >= segment_size: + chunks_to_encode = [] + # extract as many chunks as we can from the input buffer + while total_buf_len >= segment_size: + to_take = segment_size + pieces = [] + while to_take > 0: + piece = buf.popleft() + if len(piece) > to_take: + buf.appendleft(piece[to_take:]) + piece = piece[:to_take] + pieces.append(piece) + to_take -= len(piece) + total_buf_len -= len(piece) + chunks_to_encode.append(''.join(pieces)) + + frags_by_byte_order = [] + for chunk_to_encode in chunks_to_encode: + frags_by_byte_order.append( + policy.pyeclib_driver.encode(chunk_to_encode)) + # Sequential calls to encode() have given us a list that + # looks like this: + # + # [[frag_A1, frag_B1, frag_C1, ...], + # [frag_A2, frag_B2, frag_C2, ...], ...] + # + # What we need is a list like this: + # + # [(frag_A1 + frag_A2 + ...), # destined for node A + # (frag_B1 + frag_B2 + ...), # destined for node B + # (frag_C1 + frag_C2 + ...), # destined for node C + # ...] + obj_data = [''.join(frags) + for frags in zip(*frags_by_byte_order)] + chunk = yield obj_data + else: + # didn't have enough data to encode + chunk = yield None + + # Now we've gotten an empty chunk, which indicates end-of-input. + # Take any leftover bytes and encode them. + last_bytes = ''.join(buf) + if last_bytes: + last_frags = policy.pyeclib_driver.encode(last_bytes) + yield last_frags + else: + yield [''] * nstreams + + +def trailing_metadata(policy, client_obj_hasher, + bytes_transferred_from_client, + fragment_archive_index): + return { + # etag and size values are being added twice here. + # The container override header is used to update the container db + # with these values as they represent the correct etag and size for + # the whole object and not just the FA. + # The object sysmeta headers will be saved on each FA of the object. + 'X-Object-Sysmeta-EC-Etag': client_obj_hasher.hexdigest(), + 'X-Object-Sysmeta-EC-Content-Length': + str(bytes_transferred_from_client), + 'X-Backend-Container-Update-Override-Etag': + client_obj_hasher.hexdigest(), + 'X-Backend-Container-Update-Override-Size': + str(bytes_transferred_from_client), + 'X-Object-Sysmeta-Ec-Frag-Index': str(fragment_archive_index), + # These fields are for debuggability, + # AKA "what is this thing?" + 'X-Object-Sysmeta-EC-Scheme': policy.ec_scheme_description, + 'X-Object-Sysmeta-EC-Segment-Size': str(policy.ec_segment_size), + } + + +@ObjectControllerRouter.register(EC_POLICY) +class ECObjectController(BaseObjectController): + + def _get_or_head_response(self, req, node_iter, partition, policy): + req.headers.setdefault("X-Backend-Etag-Is-At", + "X-Object-Sysmeta-Ec-Etag") + + if req.method == 'HEAD': + # no fancy EC decoding here, just one plain old HEAD request to + # one object server because all fragments hold all metadata + # information about the object. + resp = self.GETorHEAD_base( + req, _('Object'), node_iter, partition, + req.swift_entity_path) + else: # GET request + orig_range = None + range_specs = [] + if req.range: + orig_range = req.range + # Since segments and fragments have different sizes, we need + # to modify the Range header sent to the object servers to + # make sure we get the right fragments out of the fragment + # archives. + segment_size = policy.ec_segment_size + fragment_size = policy.fragment_size + + range_specs = [] + new_ranges = [] + for client_start, client_end in req.range.ranges: + + segment_start, segment_end = client_range_to_segment_range( + client_start, client_end, segment_size) + + fragment_start, fragment_end = \ + segment_range_to_fragment_range( + segment_start, segment_end, + segment_size, fragment_size) + + new_ranges.append((fragment_start, fragment_end)) + range_specs.append({'client_start': client_start, + 'client_end': client_end, + 'segment_start': segment_start, + 'segment_end': segment_end}) + + req.range = "bytes=" + ",".join( + "%s-%s" % (s if s is not None else "", + e if e is not None else "") + for s, e in new_ranges) + + node_iter = GreenthreadSafeIterator(node_iter) + num_gets = policy.ec_ndata + with ContextPool(num_gets) as pool: + pile = GreenAsyncPile(pool) + for _junk in range(num_gets): + pile.spawn(self.GETorHEAD_base, + req, 'Object', node_iter, partition, + req.swift_entity_path, + client_chunk_size=policy.fragment_size) + + responses = list(pile) + good_responses = [] + bad_responses = [] + for response in responses: + if is_success(response.status_int): + good_responses.append(response) + else: + bad_responses.append(response) + + req.range = orig_range + if len(good_responses) == num_gets: + # If these aren't all for the same object, then error out so + # at least the client doesn't get garbage. We can do a lot + # better here with more work, but this'll work for now. + found_obj_etags = set( + resp.headers['X-Object-Sysmeta-Ec-Etag'] + for resp in good_responses) + if len(found_obj_etags) > 1: + self.app.logger.debug( + "Returning 503 for %s; found too many etags (%s)", + req.path, + ", ".join(found_obj_etags)) + return HTTPServiceUnavailable(request=req) + + # we found enough pieces to decode the object, so now let's + # decode the object + resp_headers = HeaderKeyDict(good_responses[0].headers.items()) + resp_headers.pop('Content-Range', None) + eccl = resp_headers.get('X-Object-Sysmeta-Ec-Content-Length') + obj_length = int(eccl) if eccl is not None else None + + resp = Response( + request=req, + headers=resp_headers, + conditional_response=True, + app_iter=ECAppIter( + req.swift_entity_path, + policy, + [r.app_iter for r in good_responses], + range_specs, + obj_length, + logger=self.app.logger)) + else: + resp = self.best_response( + req, + [r.status_int for r in bad_responses], + [r.status.split(' ', 1)[1] for r in bad_responses], + [r.body for r in bad_responses], + 'Object', + headers=[r.headers for r in bad_responses]) + + self._fix_response_headers(resp) + return resp + + def _fix_response_headers(self, resp): + # EC fragment archives each have different bytes, hence different + # etags. However, they all have the original object's etag stored in + # sysmeta, so we copy that here so the client gets it. + resp.headers['Etag'] = resp.headers.get( + 'X-Object-Sysmeta-Ec-Etag') + resp.headers['Content-Length'] = resp.headers.get( + 'X-Object-Sysmeta-Ec-Content-Length') + + return resp + + def _connect_put_node(self, node_iter, part, path, headers, + logger_thread_locals): + """ + Make a connection for a erasure encoded object. + + Connects to the first working node that it finds in node_iter and sends + over the request headers. Returns a Putter to handle the rest of the + streaming, or None if no working nodes were found. + """ + # the object server will get different bytes, so these + # values do not apply (Content-Length might, in general, but + # in the specific case of replication vs. EC, it doesn't). + headers.pop('Content-Length', None) + headers.pop('Etag', None) + + self.app.logger.thread_locals = logger_thread_locals + for node in node_iter: + try: + putter = ECPutter.connect( + node, part, path, headers, + conn_timeout=self.app.conn_timeout, + node_timeout=self.app.node_timeout) + self.app.set_node_timing(node, putter.connect_duration) + return putter + except InsufficientStorage: + self.app.error_limit(node, _('ERROR Insufficient Storage')) + except PutterConnectError as e: + self.app.error_occurred( + node, _('ERROR %(status)d Expect: 100-continue ' + 'From Object Server') % { + 'status': e.status}) + except (Exception, Timeout): + self.app.exception_occurred( + node, _('Object'), + _('Expect: 100-continue on %s') % path) + + def _determine_chunk_destinations(self, putters): + """ + Given a list of putters, return a dict where the key is the putter + and the value is the node index to use. + + This is done so that we line up handoffs using the same node index + (in the primary part list) as the primary that the handoff is standing + in for. This lets erasure-code fragment archives wind up on the + preferred local primary nodes when possible. + """ + # Give each putter a "chunk index": the index of the + # transformed chunk that we'll send to it. + # + # For primary nodes, that's just its index (primary 0 gets + # chunk 0, primary 1 gets chunk 1, and so on). For handoffs, + # we assign the chunk index of a missing primary. + handoff_conns = [] + chunk_index = {} + for p in putters: + if p.node_index is not None: + chunk_index[p] = p.node_index + else: + handoff_conns.append(p) + + # Note: we may have more holes than handoffs. This is okay; it + # just means that we failed to connect to one or more storage + # nodes. Holes occur when a storage node is down, in which + # case the connection is not replaced, and when a storage node + # returns 507, in which case a handoff is used to replace it. + holes = [x for x in range(len(putters)) + if x not in chunk_index.values()] + + for hole, p in zip(holes, handoff_conns): + chunk_index[p] = hole + return chunk_index + + def _transfer_data(self, req, policy, data_source, putters, nodes, + min_conns, etag_hasher): + """ + Transfer data for an erasure coded object. + + This method was added in the PUT method extraction change + """ + bytes_transferred = 0 + chunk_transform = chunk_transformer(policy, len(nodes)) + chunk_transform.send(None) + + def send_chunk(chunk): + if etag_hasher: + etag_hasher.update(chunk) + backend_chunks = chunk_transform.send(chunk) + if backend_chunks is None: + # If there's not enough bytes buffered for erasure-encoding + # or whatever we're doing, the transform will give us None. + return + + for putter in list(putters): + backend_chunk = backend_chunks[chunk_index[putter]] + if not putter.failed: + putter.chunk_hasher.update(backend_chunk) + putter.send_chunk(backend_chunk) + else: + putters.remove(putter) + self._check_min_conn( + req, putters, min_conns, msg='Object PUT exceptions during' + ' send, %(conns)s/%(nodes)s required connections') + + try: + with ContextPool(len(putters)) as pool: + + # build our chunk index dict to place handoffs in the + # same part nodes index as the primaries they are covering + chunk_index = self._determine_chunk_destinations(putters) + + for putter in putters: + putter.spawn_sender_greenthread( + pool, self.app.put_queue_depth, self.app.node_timeout, + self.app.exception_occurred) + while True: + with ChunkReadTimeout(self.app.client_timeout): + try: + chunk = next(data_source) + except StopIteration: + computed_etag = (etag_hasher.hexdigest() + if etag_hasher else None) + received_etag = req.headers.get( + 'etag', '').strip('"') + if (computed_etag and received_etag and + computed_etag != received_etag): + raise HTTPUnprocessableEntity(request=req) + + send_chunk('') # flush out any buffered data + + for putter in putters: + trail_md = trailing_metadata( + policy, etag_hasher, + bytes_transferred, + chunk_index[putter]) + trail_md['Etag'] = \ + putter.chunk_hasher.hexdigest() + putter.end_of_object_data(trail_md) + break + bytes_transferred += len(chunk) + if bytes_transferred > constraints.MAX_FILE_SIZE: + raise HTTPRequestEntityTooLarge(request=req) + + send_chunk(chunk) + + for putter in putters: + putter.wait() + + # for storage policies requiring 2-phase commit (e.g. + # erasure coding), enforce >= 'quorum' number of + # 100-continue responses - this indicates successful + # object data and metadata commit and is a necessary + # condition to be met before starting 2nd PUT phase + final_phase = False + need_quorum = True + statuses, reasons, bodies, _junk, quorum = \ + self._get_put_responses( + req, putters, len(nodes), final_phase, + min_conns, need_quorum=need_quorum) + if not quorum: + self.app.logger.error( + _('Not enough object servers ack\'ed (got %d)'), + statuses.count(HTTP_CONTINUE)) + raise HTTPServiceUnavailable(request=req) + # quorum achieved, start 2nd phase - send commit + # confirmation to participating object servers + # so they write a .durable state file indicating + # a successful PUT + for putter in putters: + putter.send_commit_confirmation() + for putter in putters: + putter.wait() + except ChunkReadTimeout as err: + self.app.logger.warn( + _('ERROR Client read timeout (%ss)'), err.seconds) + self.app.logger.increment('client_timeouts') + raise HTTPRequestTimeout(request=req) + except HTTPException: + raise + except (Exception, Timeout): + self.app.logger.exception( + _('ERROR Exception causing client disconnect')) + raise HTTPClientDisconnect(request=req) + if req.content_length and bytes_transferred < req.content_length: + req.client_disconnect = True + self.app.logger.warn( + _('Client disconnected without sending enough data')) + self.app.logger.increment('client_disconnects') + raise HTTPClientDisconnect(request=req) + + def _have_adequate_successes(self, statuses, min_responses): + """ + Given a list of statuses from several requests, determine if a + satisfactory number of nodes have responded with 2xx statuses to + deem the transaction for a succssful response to the client. + + :param statuses: list of statuses returned so far + :param min_responses: minimal pass criterion for number of successes + :returns: True or False, depending on current number of successes + """ + if sum(1 for s in statuses if is_success(s)) >= min_responses: + return True + return False + + def _await_response(self, conn, final_phase): + return conn.await_response( + self.app.node_timeout, not final_phase) + + def _get_conn_response(self, conn, req, final_phase, **kwargs): + try: + resp = self._await_response(conn, final_phase=final_phase, + **kwargs) + except (Exception, Timeout): + resp = None + if final_phase: + status_type = 'final' + else: + status_type = 'commit' + self.app.exception_occurred( + conn.node, _('Object'), + _('Trying to get %s status of PUT to %s') % ( + status_type, req.path)) + return (conn, resp) + + def _get_put_responses(self, req, putters, num_nodes, final_phase, + min_responses, need_quorum=True): + """ + Collect erasure coded object responses. + + Collect object responses to a PUT request and determine if + satisfactory number of nodes have returned success. Return + statuses, quorum result if indicated by 'need_quorum' and + etags if this is a final phase or a multiphase PUT transaction. + + :param req: the request + :param putters: list of putters for the request + :param num_nodes: number of nodes involved + :param final_phase: boolean indicating if this is the last phase + :param min_responses: minimum needed when not requiring quorum + :param need_quorum: boolean indicating if quorum is required + """ + statuses = [] + reasons = [] + bodies = [] + etags = set() + + pile = GreenAsyncPile(len(putters)) + for putter in putters: + if putter.failed: + continue + pile.spawn(self._get_conn_response, putter, req, + final_phase=final_phase) + + def _handle_response(putter, response): + statuses.append(response.status) + reasons.append(response.reason) + if final_phase: + body = response.read() + bodies.append(body) + else: + body = '' + if response.status == HTTP_INSUFFICIENT_STORAGE: + putter.failed = True + self.app.error_limit(putter.node, + _('ERROR Insufficient Storage')) + elif response.status >= HTTP_INTERNAL_SERVER_ERROR: + putter.failed = True + self.app.error_occurred( + putter.node, + _('ERROR %(status)d %(body)s From Object Server ' + 're: %(path)s') % + {'status': response.status, + 'body': body[:1024], 'path': req.path}) + elif is_success(response.status): + etags.add(response.getheader('etag').strip('"')) + + quorum = False + for (putter, response) in pile: + if response: + _handle_response(putter, response) + if self._have_adequate_successes(statuses, min_responses): + break + else: + putter.failed = True + + # give any pending requests *some* chance to finish + finished_quickly = pile.waitall(self.app.post_quorum_timeout) + for (putter, response) in finished_quickly: + if response: + _handle_response(putter, response) + + if need_quorum: + if final_phase: + while len(statuses) < num_nodes: + statuses.append(HTTP_SERVICE_UNAVAILABLE) + reasons.append('') + bodies.append('') + else: + # intermediate response phase - set return value to true only + # if there are enough 100-continue acknowledgements + if self.have_quorum(statuses, num_nodes): + quorum = True + + return statuses, reasons, bodies, etags, quorum + + def _store_object(self, req, data_source, nodes, partition, + outgoing_headers): + """ + Store an erasure coded object. + """ + policy_index = int(req.headers.get('X-Backend-Storage-Policy-Index')) + policy = POLICIES.get_by_index(policy_index) + # Since the request body sent from client -> proxy is not + # the same as the request body sent proxy -> object, we + # can't rely on the object-server to do the etag checking - + # so we have to do it here. + etag_hasher = md5() + + min_conns = policy.quorum + putters = self._get_put_connections( + req, nodes, partition, outgoing_headers, + policy, expect=True) + + try: + # check that a minimum number of connections were established and + # meet all the correct conditions set in the request + self._check_failure_put_connections(putters, req, nodes, min_conns) + + self._transfer_data(req, policy, data_source, putters, + nodes, min_conns, etag_hasher) + final_phase = True + need_quorum = False + min_resp = 2 + putters = [p for p in putters if not p.failed] + # ignore response etags, and quorum boolean + statuses, reasons, bodies, _etags, _quorum = \ + self._get_put_responses(req, putters, len(nodes), + final_phase, min_resp, + need_quorum=need_quorum) + except HTTPException as resp: + return resp + + etag = etag_hasher.hexdigest() + resp = self.best_response(req, statuses, reasons, bodies, + _('Object PUT'), etag=etag, + quorum_size=min_conns) + resp.last_modified = math.ceil( + float(Timestamp(req.headers['X-Timestamp']))) + return resp diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 28d41df55..8c9e22372 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -20,6 +20,8 @@ from swift import gettext_ as _ from random import shuffle from time import time import itertools +import functools +import sys from eventlet import Timeout @@ -32,11 +34,12 @@ from swift.common.utils import cache_from_env, get_logger, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \ register_swift_info from swift.common.constraints import check_utf8 -from swift.proxy.controllers import AccountController, ObjectController, \ - ContainerController, InfoController +from swift.proxy.controllers import AccountController, ContainerController, \ + ObjectControllerRouter, InfoController +from swift.proxy.controllers.base import get_container_info from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ HTTPMethodNotAllowed, HTTPNotFound, HTTPPreconditionFailed, \ - HTTPServerError, HTTPException, Request + HTTPServerError, HTTPException, Request, HTTPServiceUnavailable # List of entry points for mandatory middlewares. @@ -109,6 +112,7 @@ class Application(object): # ensure rings are loaded for all configured storage policies for policy in POLICIES: policy.load_ring(swift_dir) + self.obj_controller_router = ObjectControllerRouter() self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) @@ -235,29 +239,44 @@ class Application(object): """ return POLICIES.get_object_ring(policy_idx, self.swift_dir) - def get_controller(self, path): + def get_controller(self, req): """ Get the controller to handle a request. - :param path: path from request + :param req: the request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ - if path == '/info': + if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) return InfoController, d - version, account, container, obj = split_path(path, 1, 4, True) + version, account, container, obj = split_path(req.path, 1, 4, True) d = dict(version=version, account_name=account, container_name=container, object_name=obj) if obj and container and account: - return ObjectController, d + info = get_container_info(req.environ, self) + policy_index = req.headers.get('X-Backend-Storage-Policy-Index', + info['storage_policy']) + policy = POLICIES.get_by_index(policy_index) + if not policy: + # This indicates that a new policy has been created, + # with rings, deployed, released (i.e. deprecated = + # False), used by a client to create a container via + # another proxy that was restarted after the policy + # was released, and is now cached - all before this + # worker was HUPed to stop accepting new + # connections. There should never be an "unknown" + # index - but when there is - it's probably operator + # error and hopefully temporary. + raise HTTPServiceUnavailable('Unknown Storage Policy') + return self.obj_controller_router[policy], d elif container and account: return ContainerController, d elif account and not container and not obj: @@ -317,7 +336,7 @@ class Application(object): request=req, body='Invalid UTF8 or contains NULL') try: - controller, path_parts = self.get_controller(req.path) + controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, unicode): p = p.encode('utf-8') @@ -474,9 +493,9 @@ class Application(object): def iter_nodes(self, ring, partition, node_iter=None): """ Yields nodes for a ring partition, skipping over error - limited nodes and stopping at the configurable number of - nodes. If a node yielded subsequently gets error limited, an - extra node will be yielded to take its place. + limited nodes and stopping at the configurable number of nodes. If a + node yielded subsequently gets error limited, an extra node will be + yielded to take its place. Note that if you're going to iterate over this concurrently from multiple greenthreads, you'll want to use a @@ -527,7 +546,8 @@ class Application(object): if nodes_left <= 0: return - def exception_occurred(self, node, typ, additional_info): + def exception_occurred(self, node, typ, additional_info, + **kwargs): """ Handle logging of generic exceptions. @@ -536,11 +556,18 @@ class Application(object): :param additional_info: additional information to log """ self._incr_node_errors(node) - self.logger.exception( - _('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: ' - '%(info)s'), - {'type': typ, 'ip': node['ip'], 'port': node['port'], - 'device': node['device'], 'info': additional_info}) + if 'level' in kwargs: + log = functools.partial(self.logger.log, kwargs.pop('level')) + if 'exc_info' not in kwargs: + kwargs['exc_info'] = sys.exc_info() + else: + log = self.logger.exception + log(_('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s' + ' re: %(info)s'), { + 'type': typ, 'ip': node['ip'], 'port': + node['port'], 'device': node['device'], + 'info': additional_info + }, **kwargs) def modify_wsgi_pipeline(self, pipe): """ diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 4a8cb80bd..73e500663 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -223,7 +223,7 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir): # make policy_to_test be policy index 0 and default for the test config sp_zero_section = sp_prefix + '0' conf.add_section(sp_zero_section) - for (k, v) in policy_to_test.get_options().items(): + for (k, v) in policy_to_test.get_info(config=True).items(): conf.set(sp_zero_section, k, v) conf.set(sp_zero_section, 'default', True) diff --git a/test/functional/tests.py b/test/functional/tests.py index 626880198..b53b7f5e4 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -1317,7 +1317,12 @@ class TestFile(Base): self.assertEqual(file_types, file_types_read) def testRangedGets(self): - file_length = 10000 + # We set the file_length to a strange multiple here. This is to check + # that ranges still work in the EC case when the requested range + # spans EC segment boundaries. The 1 MiB base value is chosen because + # that's a common EC segment size. The 1.33 multiple is to ensure we + # aren't aligned on segment boundaries + file_length = int(1048576 * 1.33) range_size = file_length / 10 file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random(file_length) diff --git a/test/probe/brain.py b/test/probe/brain.py index cbb5ef7cf..9ca931aac 100644 --- a/test/probe/brain.py +++ b/test/probe/brain.py @@ -67,7 +67,7 @@ class BrainSplitter(object): __metaclass__ = meta_command def __init__(self, url, token, container_name='test', object_name='test', - server_type='container'): + server_type='container', policy=None): self.url = url self.token = token self.account = utils.split_path(urlparse(url).path, 2, 2)[1] @@ -81,9 +81,26 @@ class BrainSplitter(object): o = object_name if server_type == 'object' else None c = container_name if server_type in ('object', 'container') else None - part, nodes = ring.Ring( - '/etc/swift/%s.ring.gz' % server_type).get_nodes( - self.account, c, o) + if server_type in ('container', 'account'): + if policy: + raise TypeError('Metadata server brains do not ' + 'support specific storage policies') + self.policy = None + self.ring = ring.Ring( + '/etc/swift/%s.ring.gz' % server_type) + elif server_type == 'object': + if not policy: + raise TypeError('Object BrainSplitters need to ' + 'specify the storage policy') + self.policy = policy + policy.load_ring('/etc/swift') + self.ring = policy.object_ring + else: + raise ValueError('Unkonwn server_type: %r' % server_type) + self.server_type = server_type + + part, nodes = self.ring.get_nodes(self.account, c, o) + node_ids = [n['id'] for n in nodes] if all(n_id in node_ids for n_id in (0, 1)): self.primary_numbers = (1, 2) @@ -172,6 +189,8 @@ parser.add_option('-o', '--object', default='object-%s' % uuid.uuid4(), help='set object name') parser.add_option('-s', '--server_type', default='container', help='set server type') +parser.add_option('-P', '--policy_name', default=None, + help='set policy') def main(): @@ -186,8 +205,17 @@ def main(): return 'ERROR: unknown command %s' % cmd url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester', 'testing') + if options.server_type == 'object' and not options.policy_name: + options.policy_name = POLICIES.default.name + if options.policy_name: + options.server_type = 'object' + policy = POLICIES.get_by_name(options.policy_name) + if not policy: + return 'ERROR: unknown policy %r' % options.policy_name + else: + policy = None brain = BrainSplitter(url, token, options.container, options.object, - options.server_type) + options.server_type, policy=policy) for cmd_args in commands: parts = cmd_args.split(':', 1) command = parts[0] diff --git a/test/probe/common.py b/test/probe/common.py index 3cea02241..1311cc178 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -24,15 +24,19 @@ from nose import SkipTest from swiftclient import get_auth, head_account +from swift.obj.diskfile import get_data_dir from swift.common.ring import Ring from swift.common.utils import readconf from swift.common.manager import Manager -from swift.common.storage_policy import POLICIES +from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC ENABLED_POLICIES = [p for p in POLICIES if not p.is_deprecated] +POLICIES_BY_TYPE = defaultdict(list) +for p in POLICIES: + POLICIES_BY_TYPE[p.policy_type].append(p) def get_server_number(port, port2server): @@ -138,6 +142,17 @@ def kill_nonprimary_server(primary_nodes, port2server, pids): return port +def build_port_to_conf(server): + # map server to config by port + port_to_config = {} + for server_ in Manager([server]): + for config_path in server_.conf_files(): + conf = readconf(config_path, + section_name='%s-replicator' % server_.type) + port_to_config[int(conf['bind_port'])] = conf + return port_to_config + + def get_ring(ring_name, required_replicas, required_devices, server=None, force_validate=None): if not server: @@ -152,13 +167,7 @@ def get_ring(ring_name, required_replicas, required_devices, if len(ring.devs) != required_devices: raise SkipTest('%s has %s devices instead of %s' % ( ring.serialized_path, len(ring.devs), required_devices)) - # map server to config by port - port_to_config = {} - for server_ in Manager([server]): - for config_path in server_.conf_files(): - conf = readconf(config_path, - section_name='%s-replicator' % server_.type) - port_to_config[int(conf['bind_port'])] = conf + port_to_config = build_port_to_conf(server) for dev in ring.devs: # verify server is exposing mounted device conf = port_to_config[dev['port']] @@ -262,6 +271,10 @@ class ProbeTest(unittest.TestCase): ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) + self.server_port_to_conf = {} + # get some configs backend daemon configs loaded up + for server in ('account', 'container', 'object'): + self.server_port_to_conf[server] = build_port_to_conf(server) except BaseException: try: raise @@ -274,6 +287,18 @@ class ProbeTest(unittest.TestCase): def tearDown(self): Manager(['all']).kill() + def device_dir(self, server, node): + conf = self.server_port_to_conf[server][node['port']] + return os.path.join(conf['devices'], node['device']) + + def storage_dir(self, server, node, part=None, policy=None): + policy = policy or self.policy + device_path = self.device_dir(server, node) + path_parts = [device_path, get_data_dir(policy)] + if part is not None: + path_parts.append(str(part)) + return os.path.join(*path_parts) + def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions @@ -291,7 +316,16 @@ class ReplProbeTest(ProbeTest): acct_cont_required_devices = 4 obj_required_replicas = 3 obj_required_devices = 4 - policy_requirements = {'is_default': True} + policy_requirements = {'policy_type': REPL_POLICY} + + +class ECProbeTest(ProbeTest): + + acct_cont_required_replicas = 3 + acct_cont_required_devices = 4 + obj_required_replicas = 6 + obj_required_devices = 8 + policy_requirements = {'policy_type': EC_POLICY} if __name__ == "__main__": diff --git a/test/probe/test_container_merge_policy_index.py b/test/probe/test_container_merge_policy_index.py index dd4e50477..d604b1371 100644 --- a/test/probe/test_container_merge_policy_index.py +++ b/test/probe/test_container_merge_policy_index.py @@ -26,7 +26,8 @@ from swift.common import utils, direct_client from swift.common.storage_policy import POLICIES from swift.common.http import HTTP_NOT_FOUND from test.probe.brain import BrainSplitter -from test.probe.common import ReplProbeTest, ENABLED_POLICIES +from test.probe.common import (ReplProbeTest, ENABLED_POLICIES, + POLICIES_BY_TYPE, REPL_POLICY) from swiftclient import client, ClientException @@ -234,6 +235,18 @@ class TestContainerMergePolicyIndex(ReplProbeTest): orig_policy_index, node)) def test_reconcile_manifest(self): + # this test is not only testing a split brain scenario on + # multiple policies with mis-placed objects - it even writes out + # a static large object directly to the storage nodes while the + # objects are unavailably mis-placed from *behind* the proxy and + # doesn't know how to do that for EC_POLICY (clayg: why did you + # guys let me write a test that does this!?) - so we force + # wrong_policy (where the manifest gets written) to be one of + # any of your configured REPL_POLICY (we know you have one + # because this is a ReplProbeTest) + wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY]) + policy = random.choice([p for p in ENABLED_POLICIES + if p is not wrong_policy]) manifest_data = [] def write_part(i): @@ -250,17 +263,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest): # get an old container stashed self.brain.stop_primary_half() - policy = random.choice(ENABLED_POLICIES) - self.brain.put_container(policy.idx) + self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() - wrong_policy = random.choice([p for p in ENABLED_POLICIES - if p is not policy]) - self.brain.put_container(wrong_policy.idx) + self.brain.put_container(int(wrong_policy)) # write some more parts for i in range(10, 20): write_part(i) diff --git a/test/probe/test_empty_device_handoff.py b/test/probe/test_empty_device_handoff.py index 7002fa487..e0e450a4b 100755 --- a/test/probe/test_empty_device_handoff.py +++ b/test/probe/test_empty_device_handoff.py @@ -44,7 +44,9 @@ class TestEmptyDevice(ReplProbeTest): def test_main(self): # Create container container = 'container-%s' % uuid4() - client.put_container(self.url, self.token, container) + client.put_container(self.url, self.token, container, + headers={'X-Storage-Policy': + self.policy.name}) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] @@ -58,7 +60,7 @@ class TestEmptyDevice(ReplProbeTest): # Delete the default data directory for objects on the primary server obj_dir = '%s/%s' % (self._get_objects_dir(onode), - get_data_dir(self.policy.idx)) + get_data_dir(self.policy)) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index 34ec08253..05d05b3ad 100755 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -108,7 +108,9 @@ class TestUpdateOverrides(ReplProbeTest): 'X-Backend-Container-Update-Override-Etag': 'override-etag', 'X-Backend-Container-Update-Override-Content-Type': 'override-type' } - client.put_container(self.url, self.token, 'c1') + client.put_container(self.url, self.token, 'c1', + headers={'X-Storage-Policy': + self.policy.name}) self.int_client.upload_object(StringIO(u'stuff'), self.account, 'c1', 'o1', headers) diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index 9147b1ed5..469683a10 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -52,7 +52,9 @@ def get_data_file_path(obj_dir): class TestObjectFailures(ReplProbeTest): def _setup_data_file(self, container, obj, data): - client.put_container(self.url, self.token, container) + client.put_container(self.url, self.token, container, + headers={'X-Storage-Policy': + self.policy.name}) client.put_object(self.url, self.token, container, obj, data) odata = client.get_object(self.url, self.token, container, obj)[-1] self.assertEquals(odata, data) @@ -65,7 +67,7 @@ class TestObjectFailures(ReplProbeTest): obj_server_conf = readconf(self.configs['object-server'][node_id]) devices = obj_server_conf['app:object-server']['devices'] obj_dir = '%s/%s/%s/%s/%s/%s/' % (devices, device, - get_data_dir(self.policy.idx), + get_data_dir(self.policy), opart, hash_str[-3:], hash_str) data_file = get_data_file_path(obj_dir) return onode, opart, data_file diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 41a67cf28..f513eef2e 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -30,7 +30,9 @@ class TestObjectHandoff(ReplProbeTest): def test_main(self): # Create container container = 'container-%s' % uuid4() - client.put_container(self.url, self.token, container) + client.put_container(self.url, self.token, container, + headers={'X-Storage-Policy': + self.policy.name}) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) diff --git a/test/probe/test_object_metadata_replication.py b/test/probe/test_object_metadata_replication.py index 357cfec5b..c278e5f81 100644 --- a/test/probe/test_object_metadata_replication.py +++ b/test/probe/test_object_metadata_replication.py @@ -73,7 +73,8 @@ class Test(ReplProbeTest): self.container_name = 'container-%s' % uuid.uuid4() self.object_name = 'object-%s' % uuid.uuid4() self.brain = BrainSplitter(self.url, self.token, self.container_name, - self.object_name, 'object') + self.object_name, 'object', + policy=self.policy) self.tempdir = mkdtemp() conf_path = os.path.join(self.tempdir, 'internal_client.conf') conf_body = """ @@ -128,7 +129,7 @@ class Test(ReplProbeTest): self.object_name) def test_object_delete_is_replicated(self): - self.brain.put_container(policy_index=0) + self.brain.put_container(policy_index=int(self.policy)) # put object self._put_object() @@ -174,7 +175,7 @@ class Test(ReplProbeTest): def test_sysmeta_after_replication_with_subsequent_post(self): sysmeta = {'x-object-sysmeta-foo': 'sysmeta-foo'} usermeta = {'x-object-meta-bar': 'meta-bar'} - self.brain.put_container(policy_index=0) + self.brain.put_container(policy_index=int(self.policy)) # put object self._put_object() # put newer object with sysmeta to first server subset @@ -221,7 +222,7 @@ class Test(ReplProbeTest): def test_sysmeta_after_replication_with_prior_post(self): sysmeta = {'x-object-sysmeta-foo': 'sysmeta-foo'} usermeta = {'x-object-meta-bar': 'meta-bar'} - self.brain.put_container(policy_index=0) + self.brain.put_container(policy_index=int(self.policy)) # put object self._put_object() diff --git a/test/probe/test_reconstructor_durable.py b/test/probe/test_reconstructor_durable.py new file mode 100644 index 000000000..eeef00e62 --- /dev/null +++ b/test/probe/test_reconstructor_durable.py @@ -0,0 +1,157 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hashlib import md5 +import unittest +import uuid +import random +import os +import errno + +from test.probe.common import ECProbeTest + +from swift.common import direct_client +from swift.common.storage_policy import EC_POLICY +from swift.common.manager import Manager + +from swiftclient import client + + +class Body(object): + + def __init__(self, total=3.5 * 2 ** 20): + self.total = total + self.hasher = md5() + self.size = 0 + self.chunk = 'test' * 16 * 2 ** 10 + + @property + def etag(self): + return self.hasher.hexdigest() + + def __iter__(self): + return self + + def next(self): + if self.size > self.total: + raise StopIteration() + self.size += len(self.chunk) + self.hasher.update(self.chunk) + return self.chunk + + def __next__(self): + return self.next() + + +class TestReconstructorPropDurable(ECProbeTest): + + def setUp(self): + super(TestReconstructorPropDurable, self).setUp() + self.container_name = 'container-%s' % uuid.uuid4() + self.object_name = 'object-%s' % uuid.uuid4() + # sanity + self.assertEqual(self.policy.policy_type, EC_POLICY) + self.reconstructor = Manager(["object-reconstructor"]) + + def direct_get(self, node, part): + req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)} + headers, data = direct_client.direct_get_object( + node, part, self.account, self.container_name, + self.object_name, headers=req_headers, + resp_chunk_size=64 * 2 ** 20) + hasher = md5() + for chunk in data: + hasher.update(chunk) + return hasher.hexdigest() + + def _check_node(self, node, part, etag, headers_post): + # get fragment archive etag + fragment_archive_etag = self.direct_get(node, part) + + # remove the .durable from the selected node + part_dir = self.storage_dir('object', node, part=part) + for dirs, subdirs, files in os.walk(part_dir): + for fname in files: + if fname.endswith('.durable'): + durable = os.path.join(dirs, fname) + os.remove(durable) + break + try: + os.remove(os.path.join(part_dir, 'hashes.pkl')) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + # fire up reconstructor to propogate the .durable + self.reconstructor.once() + + # fragment is still exactly as it was before! + self.assertEqual(fragment_archive_etag, + self.direct_get(node, part)) + + # check meta + meta = client.head_object(self.url, self.token, + self.container_name, + self.object_name) + for key in headers_post: + self.assertTrue(key in meta) + self.assertEqual(meta[key], headers_post[key]) + + def _format_node(self, node): + return '%s#%s' % (node['device'], node['index']) + + def test_main(self): + # create EC container + headers = {'X-Storage-Policy': self.policy.name} + client.put_container(self.url, self.token, self.container_name, + headers=headers) + + # PUT object + contents = Body() + headers = {'x-object-meta-foo': 'meta-foo'} + headers_post = {'x-object-meta-bar': 'meta-bar'} + + etag = client.put_object(self.url, self.token, + self.container_name, + self.object_name, + contents=contents, headers=headers) + client.post_object(self.url, self.token, self.container_name, + self.object_name, headers=headers_post) + del headers_post['X-Auth-Token'] # WTF, where did this come from? + + # built up a list of node lists to kill a .durable from, + # first try a single node + # then adjacent nodes and then nodes >1 node apart + opart, onodes = self.object_ring.get_nodes( + self.account, self.container_name, self.object_name) + single_node = [random.choice(onodes)] + adj_nodes = [onodes[0], onodes[-1]] + far_nodes = [onodes[0], onodes[-2]] + test_list = [single_node, adj_nodes, far_nodes] + + for node_list in test_list: + for onode in node_list: + try: + self._check_node(onode, opart, etag, headers_post) + except AssertionError as e: + self.fail( + str(e) + '\n... for node %r of scenario %r' % ( + self._format_node(onode), + [self._format_node(n) for n in node_list])) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/probe/test_reconstructor_rebuild.py b/test/probe/test_reconstructor_rebuild.py new file mode 100644 index 000000000..5edfcc52d --- /dev/null +++ b/test/probe/test_reconstructor_rebuild.py @@ -0,0 +1,170 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hashlib import md5 +import unittest +import uuid +import shutil +import random + +from test.probe.common import ECProbeTest + +from swift.common import direct_client +from swift.common.storage_policy import EC_POLICY +from swift.common.manager import Manager + +from swiftclient import client + + +class Body(object): + + def __init__(self, total=3.5 * 2 ** 20): + self.total = total + self.hasher = md5() + self.size = 0 + self.chunk = 'test' * 16 * 2 ** 10 + + @property + def etag(self): + return self.hasher.hexdigest() + + def __iter__(self): + return self + + def next(self): + if self.size > self.total: + raise StopIteration() + self.size += len(self.chunk) + self.hasher.update(self.chunk) + return self.chunk + + def __next__(self): + return self.next() + + +class TestReconstructorRebuild(ECProbeTest): + + def setUp(self): + super(TestReconstructorRebuild, self).setUp() + self.container_name = 'container-%s' % uuid.uuid4() + self.object_name = 'object-%s' % uuid.uuid4() + # sanity + self.assertEqual(self.policy.policy_type, EC_POLICY) + self.reconstructor = Manager(["object-reconstructor"]) + + def proxy_get(self): + # GET object + headers, body = client.get_object(self.url, self.token, + self.container_name, + self.object_name, + resp_chunk_size=64 * 2 ** 10) + resp_checksum = md5() + for chunk in body: + resp_checksum.update(chunk) + return resp_checksum.hexdigest() + + def direct_get(self, node, part): + req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)} + headers, data = direct_client.direct_get_object( + node, part, self.account, self.container_name, + self.object_name, headers=req_headers, + resp_chunk_size=64 * 2 ** 20) + hasher = md5() + for chunk in data: + hasher.update(chunk) + return hasher.hexdigest() + + def _check_node(self, node, part, etag, headers_post): + # get fragment archive etag + fragment_archive_etag = self.direct_get(node, part) + + # remove data from the selected node + part_dir = self.storage_dir('object', node, part=part) + shutil.rmtree(part_dir, True) + + # this node can't servce the data any more + try: + self.direct_get(node, part) + except direct_client.DirectClientException as err: + self.assertEqual(err.http_status, 404) + else: + self.fail('Node data on %r was not fully destoryed!' % + (node,)) + + # make sure we can still GET the object and its correct, the + # proxy is doing decode on remaining fragments to get the obj + self.assertEqual(etag, self.proxy_get()) + + # fire up reconstructor + self.reconstructor.once() + + # fragment is rebuilt exactly as it was before! + self.assertEqual(fragment_archive_etag, + self.direct_get(node, part)) + + # check meta + meta = client.head_object(self.url, self.token, + self.container_name, + self.object_name) + for key in headers_post: + self.assertTrue(key in meta) + self.assertEqual(meta[key], headers_post[key]) + + def _format_node(self, node): + return '%s#%s' % (node['device'], node['index']) + + def test_main(self): + # create EC container + headers = {'X-Storage-Policy': self.policy.name} + client.put_container(self.url, self.token, self.container_name, + headers=headers) + + # PUT object + contents = Body() + headers = {'x-object-meta-foo': 'meta-foo'} + headers_post = {'x-object-meta-bar': 'meta-bar'} + + etag = client.put_object(self.url, self.token, + self.container_name, + self.object_name, + contents=contents, headers=headers) + client.post_object(self.url, self.token, self.container_name, + self.object_name, headers=headers_post) + del headers_post['X-Auth-Token'] # WTF, where did this come from? + + # built up a list of node lists to kill data from, + # first try a single node + # then adjacent nodes and then nodes >1 node apart + opart, onodes = self.object_ring.get_nodes( + self.account, self.container_name, self.object_name) + single_node = [random.choice(onodes)] + adj_nodes = [onodes[0], onodes[-1]] + far_nodes = [onodes[0], onodes[-2]] + test_list = [single_node, adj_nodes, far_nodes] + + for node_list in test_list: + for onode in node_list: + try: + self._check_node(onode, opart, etag, headers_post) + except AssertionError as e: + self.fail( + str(e) + '\n... for node %r of scenario %r' % ( + self._format_node(onode), + [self._format_node(n) for n in node_list])) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/probe/test_reconstructor_revert.py b/test/probe/test_reconstructor_revert.py new file mode 100755 index 000000000..2a7bd7c83 --- /dev/null +++ b/test/probe/test_reconstructor_revert.py @@ -0,0 +1,258 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hashlib import md5 +import unittest +import uuid +import os + +from test.probe.common import ECProbeTest + +from swift.common import direct_client +from swift.common.storage_policy import EC_POLICY +from swift.common.manager import Manager +from swift.common.utils import renamer + +from swiftclient import client + + +class Body(object): + + def __init__(self, total=3.5 * 2 ** 20): + self.total = total + self.hasher = md5() + self.size = 0 + self.chunk = 'test' * 16 * 2 ** 10 + + @property + def etag(self): + return self.hasher.hexdigest() + + def __iter__(self): + return self + + def next(self): + if self.size > self.total: + raise StopIteration() + self.size += len(self.chunk) + self.hasher.update(self.chunk) + return self.chunk + + def __next__(self): + return self.next() + + +class TestReconstructorRevert(ECProbeTest): + + def setUp(self): + super(TestReconstructorRevert, self).setUp() + self.container_name = 'container-%s' % uuid.uuid4() + self.object_name = 'object-%s' % uuid.uuid4() + + # sanity + self.assertEqual(self.policy.policy_type, EC_POLICY) + self.reconstructor = Manager(["object-reconstructor"]) + + def kill_drive(self, device): + if os.path.ismount(device): + os.system('sudo umount %s' % device) + else: + renamer(device, device + "X") + + def revive_drive(self, device): + disabled_name = device + "X" + if os.path.isdir(disabled_name): + renamer(device + "X", device) + else: + os.system('sudo mount %s' % device) + + def proxy_get(self): + # GET object + headers, body = client.get_object(self.url, self.token, + self.container_name, + self.object_name, + resp_chunk_size=64 * 2 ** 10) + resp_checksum = md5() + for chunk in body: + resp_checksum.update(chunk) + return resp_checksum.hexdigest() + + def direct_get(self, node, part): + req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)} + headers, data = direct_client.direct_get_object( + node, part, self.account, self.container_name, + self.object_name, headers=req_headers, + resp_chunk_size=64 * 2 ** 20) + hasher = md5() + for chunk in data: + hasher.update(chunk) + return hasher.hexdigest() + + def test_revert_object(self): + # create EC container + headers = {'X-Storage-Policy': self.policy.name} + client.put_container(self.url, self.token, self.container_name, + headers=headers) + + # get our node lists + opart, onodes = self.object_ring.get_nodes( + self.account, self.container_name, self.object_name) + hnodes = self.object_ring.get_more_nodes(opart) + + # kill 2 a parity count number of primary nodes so we can + # force data onto handoffs, we do that by renaming dev dirs + # to induce 507 + p_dev1 = self.device_dir('object', onodes[0]) + p_dev2 = self.device_dir('object', onodes[1]) + self.kill_drive(p_dev1) + self.kill_drive(p_dev2) + + # PUT object + contents = Body() + headers = {'x-object-meta-foo': 'meta-foo'} + headers_post = {'x-object-meta-bar': 'meta-bar'} + client.put_object(self.url, self.token, self.container_name, + self.object_name, contents=contents, + headers=headers) + client.post_object(self.url, self.token, self.container_name, + self.object_name, headers=headers_post) + del headers_post['X-Auth-Token'] # WTF, where did this come from? + + # these primaries can't servce the data any more, we expect 507 + # here and not 404 because we're using mount_check to kill nodes + for onode in (onodes[0], onodes[1]): + try: + self.direct_get(onode, opart) + except direct_client.DirectClientException as err: + self.assertEqual(err.http_status, 507) + else: + self.fail('Node data on %r was not fully destoryed!' % + (onode,)) + + # now take out another primary + p_dev3 = self.device_dir('object', onodes[2]) + self.kill_drive(p_dev3) + + # this node can't servce the data any more + try: + self.direct_get(onodes[2], opart) + except direct_client.DirectClientException as err: + self.assertEqual(err.http_status, 507) + else: + self.fail('Node data on %r was not fully destoryed!' % + (onode,)) + + # make sure we can still GET the object and its correct + # we're now pulling from handoffs and reconstructing + etag = self.proxy_get() + self.assertEqual(etag, contents.etag) + + # rename the dev dirs so they don't 507 anymore + self.revive_drive(p_dev1) + self.revive_drive(p_dev2) + self.revive_drive(p_dev3) + + # fire up reconstructor on handoff nodes only + for hnode in hnodes: + hnode_id = (hnode['port'] - 6000) / 10 + self.reconstructor.once(number=hnode_id) + + # first threee primaries have data again + for onode in (onodes[0], onodes[2]): + self.direct_get(onode, opart) + + # check meta + meta = client.head_object(self.url, self.token, + self.container_name, + self.object_name) + for key in headers_post: + self.assertTrue(key in meta) + self.assertEqual(meta[key], headers_post[key]) + + # handoffs are empty + for hnode in hnodes: + try: + self.direct_get(hnode, opart) + except direct_client.DirectClientException as err: + self.assertEqual(err.http_status, 404) + else: + self.fail('Node data on %r was not fully destoryed!' % + (hnode,)) + + def test_delete_propogate(self): + # create EC container + headers = {'X-Storage-Policy': self.policy.name} + client.put_container(self.url, self.token, self.container_name, + headers=headers) + + # get our node lists + opart, onodes = self.object_ring.get_nodes( + self.account, self.container_name, self.object_name) + hnodes = self.object_ring.get_more_nodes(opart) + p_dev2 = self.device_dir('object', onodes[1]) + + # PUT object + contents = Body() + client.put_object(self.url, self.token, self.container_name, + self.object_name, contents=contents) + + # now lets shut one down + self.kill_drive(p_dev2) + + # delete on the ones that are left + client.delete_object(self.url, self.token, + self.container_name, + self.object_name) + + # spot check a node + try: + self.direct_get(onodes[0], opart) + except direct_client.DirectClientException as err: + self.assertEqual(err.http_status, 404) + else: + self.fail('Node data on %r was not fully destoryed!' % + (onodes[0],)) + + # enable the first node again + self.revive_drive(p_dev2) + + # propogate the delete... + # fire up reconstructor on handoff nodes only + for hnode in hnodes: + hnode_id = (hnode['port'] - 6000) / 10 + self.reconstructor.once(number=hnode_id, override_devices=['sdb8']) + + # check the first node to make sure its gone + try: + self.direct_get(onodes[1], opart) + except direct_client.DirectClientException as err: + self.assertEqual(err.http_status, 404) + else: + self.fail('Node data on %r was not fully destoryed!' % + (onodes[0])) + + # make sure proxy get can't find it + try: + self.proxy_get() + except Exception as err: + self.assertEqual(err.http_status, 404) + else: + self.fail('Node data on %r was not fully destoryed!' % + (onodes[0])) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/probe/test_replication_servers_working.py b/test/probe/test_replication_servers_working.py index db657f3e7..64b906fdc 100644 --- a/test/probe/test_replication_servers_working.py +++ b/test/probe/test_replication_servers_working.py @@ -21,7 +21,6 @@ import time import shutil from swiftclient import client -from swift.common.storage_policy import POLICIES from swift.obj.diskfile import get_data_dir from test.probe.common import ReplProbeTest @@ -88,7 +87,7 @@ class TestReplicatorFunctions(ReplProbeTest): # Delete file "hashes.pkl". # Check, that all files were replicated. path_list = [] - data_dir = get_data_dir(POLICIES.default.idx) + data_dir = get_data_dir(self.policy) # Figure out where the devices are for node_id in range(1, 5): conf = readconf(self.configs['object-server'][node_id]) @@ -100,7 +99,9 @@ class TestReplicatorFunctions(ReplProbeTest): # Put data to storage nodes container = 'container-%s' % uuid4() - client.put_container(self.url, self.token, container) + client.put_container(self.url, self.token, container, + headers={'X-Storage-Policy': + self.policy.name}) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, 'VERIFY') diff --git a/test/unit/__init__.py b/test/unit/__init__.py index da7212c98..372fb58bb 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -22,24 +22,30 @@ import errno import sys from contextlib import contextmanager, closing from collections import defaultdict, Iterable +import itertools from numbers import Number from tempfile import NamedTemporaryFile import time +import eventlet from eventlet.green import socket from tempfile import mkdtemp from shutil import rmtree +from swift.common.utils import Timestamp from test import get_config from swift.common import swob, utils from swift.common.ring import Ring, RingData from hashlib import md5 -from eventlet import sleep, Timeout import logging.handlers from httplib import HTTPException from swift.common import storage_policy +from swift.common.storage_policy import StoragePolicy, ECStoragePolicy import functools import cPickle as pickle from gzip import GzipFile import mock as mocklib +import inspect + +EMPTY_ETAG = md5().hexdigest() # try not to import this module from swift if not os.path.basename(sys.argv[0]).startswith('swift'): @@ -47,26 +53,40 @@ if not os.path.basename(sys.argv[0]).startswith('swift'): utils.HASH_PATH_SUFFIX = 'endcap' -def patch_policies(thing_or_policies=None, legacy_only=False): +def patch_policies(thing_or_policies=None, legacy_only=False, + with_ec_default=False, fake_ring_args=None): + if isinstance(thing_or_policies, ( + Iterable, storage_policy.StoragePolicyCollection)): + return PatchPolicies(thing_or_policies, fake_ring_args=fake_ring_args) + if legacy_only: - default_policies = [storage_policy.StoragePolicy( - 0, 'legacy', True, object_ring=FakeRing())] + default_policies = [ + StoragePolicy(0, name='legacy', is_default=True), + ] + default_ring_args = [{}] + elif with_ec_default: + default_policies = [ + ECStoragePolicy(0, name='ec', is_default=True, + ec_type='jerasure_rs_vand', ec_ndata=10, + ec_nparity=4, ec_segment_size=4096), + StoragePolicy(1, name='unu'), + ] + default_ring_args = [{'replicas': 14}, {}] else: default_policies = [ - storage_policy.StoragePolicy( - 0, 'nulo', True, object_ring=FakeRing()), - storage_policy.StoragePolicy( - 1, 'unu', object_ring=FakeRing()), + StoragePolicy(0, name='nulo', is_default=True), + StoragePolicy(1, name='unu'), ] + default_ring_args = [{}, {}] - thing_or_policies = thing_or_policies or default_policies + fake_ring_args = fake_ring_args or default_ring_args + decorator = PatchPolicies(default_policies, fake_ring_args=fake_ring_args) - if isinstance(thing_or_policies, ( - Iterable, storage_policy.StoragePolicyCollection)): - return PatchPolicies(thing_or_policies) + if not thing_or_policies: + return decorator else: - # it's a thing! - return PatchPolicies(default_policies)(thing_or_policies) + # it's a thing, we return the wrapped thing instead of the decorator + return decorator(thing_or_policies) class PatchPolicies(object): @@ -76,11 +96,33 @@ class PatchPolicies(object): patched yet) """ - def __init__(self, policies): + def __init__(self, policies, fake_ring_args=None): if isinstance(policies, storage_policy.StoragePolicyCollection): self.policies = policies else: self.policies = storage_policy.StoragePolicyCollection(policies) + self.fake_ring_args = fake_ring_args or [None] * len(self.policies) + + def _setup_rings(self): + """ + Our tests tend to use the policies rings like their own personal + playground - which can be a problem in the particular case of a + patched TestCase class where the FakeRing objects are scoped in the + call to the patch_policies wrapper outside of the TestCase instance + which can lead to some bled state. + + To help tests get better isolation without having to think about it, + here we're capturing the args required to *build* a new FakeRing + instances so we can ensure each test method gets a clean ring setup. + + The TestCase can always "tweak" these fresh rings in setUp - or if + they'd prefer to get the same "reset" behavior with custom FakeRing's + they can pass in their own fake_ring_args to patch_policies instead of + setting the object_ring on the policy definitions. + """ + for policy, fake_ring_arg in zip(self.policies, self.fake_ring_args): + if fake_ring_arg is not None: + policy.object_ring = FakeRing(**fake_ring_arg) def __call__(self, thing): if isinstance(thing, type): @@ -89,24 +131,33 @@ class PatchPolicies(object): return self._patch_method(thing) def _patch_class(self, cls): + """ + Creating a new class that inherits from decorated class is the more + common way I've seen class decorators done - but it seems to cause + infinite recursion when super is called from inside methods in the + decorated class. + """ - class NewClass(cls): + orig_setUp = cls.setUp + orig_tearDown = cls.tearDown - already_patched = False + def setUp(cls_self): + self._orig_POLICIES = storage_policy._POLICIES + if not getattr(cls_self, '_policies_patched', False): + storage_policy._POLICIES = self.policies + self._setup_rings() + cls_self._policies_patched = True - def setUp(cls_self): - self._orig_POLICIES = storage_policy._POLICIES - if not cls_self.already_patched: - storage_policy._POLICIES = self.policies - cls_self.already_patched = True - super(NewClass, cls_self).setUp() + orig_setUp(cls_self) - def tearDown(cls_self): - super(NewClass, cls_self).tearDown() - storage_policy._POLICIES = self._orig_POLICIES + def tearDown(cls_self): + orig_tearDown(cls_self) + storage_policy._POLICIES = self._orig_POLICIES - NewClass.__name__ = cls.__name__ - return NewClass + cls.setUp = setUp + cls.tearDown = tearDown + + return cls def _patch_method(self, f): @functools.wraps(f) @@ -114,6 +165,7 @@ class PatchPolicies(object): self._orig_POLICIES = storage_policy._POLICIES try: storage_policy._POLICIES = self.policies + self._setup_rings() return f(*args, **kwargs) finally: storage_policy._POLICIES = self._orig_POLICIES @@ -171,14 +223,16 @@ class FakeRing(Ring): return self.replicas def _get_part_nodes(self, part): - return list(self._devs) + return [dict(node, index=i) for i, node in enumerate(list(self._devs))] def get_more_nodes(self, part): # replicas^2 is the true cap for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes, self.replicas * self.replicas)): yield {'ip': '10.0.0.%s' % x, + 'replication_ip': '10.0.0.%s' % x, 'port': self._base_port + x, + 'replication_port': self._base_port + x, 'device': 'sda', 'zone': x % 3, 'region': x % 2, @@ -206,6 +260,48 @@ def write_fake_ring(path, *devs): pickle.dump(RingData(replica2part2dev_id, devs, part_shift), f) +class FabricatedRing(Ring): + """ + When a FakeRing just won't do - you can fabricate one to meet + your tests needs. + """ + + def __init__(self, replicas=6, devices=8, nodes=4, port=6000, + part_power=4): + self.devices = devices + self.nodes = nodes + self.port = port + self.replicas = 6 + self.part_power = part_power + self._part_shift = 32 - self.part_power + self._reload() + + def _reload(self, *args, **kwargs): + self._rtime = time.time() * 2 + if hasattr(self, '_replica2part2dev_id'): + return + self._devs = [{ + 'region': 1, + 'zone': 1, + 'weight': 1.0, + 'id': i, + 'device': 'sda%d' % i, + 'ip': '10.0.0.%d' % (i % self.nodes), + 'replication_ip': '10.0.0.%d' % (i % self.nodes), + 'port': self.port, + 'replication_port': self.port, + } for i in range(self.devices)] + + self._replica2part2dev_id = [ + [None] * 2 ** self.part_power + for i in range(self.replicas) + ] + dev_ids = itertools.cycle(range(self.devices)) + for p in range(2 ** self.part_power): + for r in range(self.replicas): + self._replica2part2dev_id[r][p] = next(dev_ids) + + class FakeMemcache(object): def __init__(self): @@ -363,8 +459,8 @@ class UnmockTimeModule(object): logging.time = UnmockTimeModule() -class FakeLogger(logging.Logger): - # a thread safe logger +class FakeLogger(logging.Logger, object): + # a thread safe fake logger def __init__(self, *args, **kwargs): self._clear() @@ -376,22 +472,31 @@ class FakeLogger(logging.Logger): self.thread_locals = None self.parent = None + store_in = { + logging.ERROR: 'error', + logging.WARNING: 'warning', + logging.INFO: 'info', + logging.DEBUG: 'debug', + logging.CRITICAL: 'critical', + } + + def _log(self, level, msg, *args, **kwargs): + store_name = self.store_in[level] + cargs = [msg] + if any(args): + cargs.extend(args) + captured = dict(kwargs) + if 'exc_info' in kwargs and \ + not isinstance(kwargs['exc_info'], tuple): + captured['exc_info'] = sys.exc_info() + self.log_dict[store_name].append((tuple(cargs), captured)) + super(FakeLogger, self)._log(level, msg, *args, **kwargs) + def _clear(self): self.log_dict = defaultdict(list) self.lines_dict = {'critical': [], 'error': [], 'info': [], 'warning': [], 'debug': []} - def _store_in(store_name): - def stub_fn(self, *args, **kwargs): - self.log_dict[store_name].append((args, kwargs)) - return stub_fn - - def _store_and_log_in(store_name, level): - def stub_fn(self, *args, **kwargs): - self.log_dict[store_name].append((args, kwargs)) - self._log(level, args[0], args[1:], **kwargs) - return stub_fn - def get_lines_for_level(self, level): if level not in self.lines_dict: raise KeyError( @@ -404,16 +509,10 @@ class FakeLogger(logging.Logger): return dict((level, msgs) for level, msgs in self.lines_dict.items() if len(msgs) > 0) - error = _store_and_log_in('error', logging.ERROR) - info = _store_and_log_in('info', logging.INFO) - warning = _store_and_log_in('warning', logging.WARNING) - warn = _store_and_log_in('warning', logging.WARNING) - debug = _store_and_log_in('debug', logging.DEBUG) - - def exception(self, *args, **kwargs): - self.log_dict['exception'].append((args, kwargs, - str(sys.exc_info()[1]))) - print 'FakeLogger Exception: %s' % self.log_dict + def _store_in(store_name): + def stub_fn(self, *args, **kwargs): + self.log_dict[store_name].append((args, kwargs)) + return stub_fn # mock out the StatsD logging methods: update_stats = _store_in('update_stats') @@ -605,19 +704,53 @@ def mock(update): delattr(module, attr) +class SlowBody(object): + """ + This will work with our fake_http_connect, if you hand in these + instead of strings it will make reads take longer by the given + amount. It should be a little bit easier to extend than the + current slow kwarg - which inserts whitespace in the response. + Also it should be easy to detect if you have one of these (or a + subclass) for the body inside of FakeConn if we wanted to do + something smarter than just duck-type the str/buffer api + enough to get by. + """ + + def __init__(self, body, slowness): + self.body = body + self.slowness = slowness + + def slowdown(self): + eventlet.sleep(self.slowness) + + def __getitem__(self, s): + return SlowBody(self.body[s], self.slowness) + + def __len__(self): + return len(self.body) + + def __radd__(self, other): + self.slowdown() + return other + self.body + + def fake_http_connect(*code_iter, **kwargs): class FakeConn(object): def __init__(self, status, etag=None, body='', timestamp='1', - headers=None): + headers=None, expect_headers=None, connection_id=None, + give_send=None): # connect exception - if isinstance(status, (Exception, Timeout)): + if isinstance(status, (Exception, eventlet.Timeout)): raise status if isinstance(status, tuple): - self.expect_status, self.status = status + self.expect_status = list(status[:-1]) + self.status = status[-1] + self.explicit_expect_list = True else: - self.expect_status, self.status = (None, status) + self.expect_status, self.status = ([], status) + self.explicit_expect_list = False if not self.expect_status: # when a swift backend service returns a status before reading # from the body (mostly an error response) eventlet.wsgi will @@ -628,9 +761,9 @@ def fake_http_connect(*code_iter, **kwargs): # our backend services and return certain types of responses # as expect statuses just like a real backend server would do. if self.status in (507, 412, 409): - self.expect_status = status + self.expect_status = [status] else: - self.expect_status = 100 + self.expect_status = [100, 100] self.reason = 'Fake' self.host = '1.2.3.4' self.port = '1234' @@ -639,32 +772,41 @@ def fake_http_connect(*code_iter, **kwargs): self.etag = etag self.body = body self.headers = headers or {} + self.expect_headers = expect_headers or {} self.timestamp = timestamp + self.connection_id = connection_id + self.give_send = give_send if 'slow' in kwargs and isinstance(kwargs['slow'], list): try: self._next_sleep = kwargs['slow'].pop(0) except IndexError: self._next_sleep = None + # be nice to trixy bits with node_iter's + eventlet.sleep() def getresponse(self): - if isinstance(self.status, (Exception, Timeout)): + if self.expect_status and self.explicit_expect_list: + raise Exception('Test did not consume all fake ' + 'expect status: %r' % (self.expect_status,)) + if isinstance(self.status, (Exception, eventlet.Timeout)): raise self.status exc = kwargs.get('raise_exc') if exc: - if isinstance(exc, (Exception, Timeout)): + if isinstance(exc, (Exception, eventlet.Timeout)): raise exc raise Exception('test') if kwargs.get('raise_timeout_exc'): - raise Timeout() + raise eventlet.Timeout() return self def getexpect(self): - if isinstance(self.expect_status, (Exception, Timeout)): + expect_status = self.expect_status.pop(0) + if isinstance(self.expect_status, (Exception, eventlet.Timeout)): raise self.expect_status - headers = {} - if self.expect_status == 409: + headers = dict(self.expect_headers) + if expect_status == 409: headers['X-Backend-Timestamp'] = self.timestamp - return FakeConn(self.expect_status, headers=headers) + return FakeConn(expect_status, headers=headers) def getheaders(self): etag = self.etag @@ -717,18 +859,20 @@ def fake_http_connect(*code_iter, **kwargs): if am_slow: if self.sent < 4: self.sent += 1 - sleep(value) + eventlet.sleep(value) return ' ' rv = self.body[:amt] self.body = self.body[amt:] return rv def send(self, amt=None): + if self.give_send: + self.give_send(self.connection_id, amt) am_slow, value = self.get_slow() if am_slow: if self.received < 4: self.received += 1 - sleep(value) + eventlet.sleep(value) def getheader(self, name, default=None): return swob.HeaderKeyDict(self.getheaders()).get(name, default) @@ -738,16 +882,22 @@ def fake_http_connect(*code_iter, **kwargs): timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) - if isinstance(kwargs.get('headers'), list): + if isinstance(kwargs.get('headers'), (list, tuple)): headers_iter = iter(kwargs['headers']) else: headers_iter = iter([kwargs.get('headers', {})] * len(code_iter)) + if isinstance(kwargs.get('expect_headers'), (list, tuple)): + expect_headers_iter = iter(kwargs['expect_headers']) + else: + expect_headers_iter = iter([kwargs.get('expect_headers', {})] * + len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) if not isinstance(x, (tuple, list)): x = [x] * len(code_iter) container_ts_iter = iter(x) code_iter = iter(code_iter) + conn_id_and_code_iter = enumerate(code_iter) static_body = kwargs.get('body', None) body_iter = kwargs.get('body_iter', None) if body_iter: @@ -755,17 +905,22 @@ def fake_http_connect(*code_iter, **kwargs): def connect(*args, **ckwargs): if kwargs.get('slow_connect', False): - sleep(0.1) + eventlet.sleep(0.1) if 'give_content_type' in kwargs: if len(args) >= 7 and 'Content-Type' in args[6]: kwargs['give_content_type'](args[6]['Content-Type']) else: kwargs['give_content_type']('') + i, status = conn_id_and_code_iter.next() if 'give_connect' in kwargs: - kwargs['give_connect'](*args, **ckwargs) - status = code_iter.next() + give_conn_fn = kwargs['give_connect'] + argspec = inspect.getargspec(give_conn_fn) + if argspec.keywords or 'connection_id' in argspec.args: + ckwargs['connection_id'] = i + give_conn_fn(*args, **ckwargs) etag = etag_iter.next() headers = headers_iter.next() + expect_headers = expect_headers_iter.next() timestamp = timestamps_iter.next() if status <= 0: @@ -775,7 +930,8 @@ def fake_http_connect(*code_iter, **kwargs): else: body = body_iter.next() return FakeConn(status, etag, body=body, timestamp=timestamp, - headers=headers) + headers=headers, expect_headers=expect_headers, + connection_id=i, give_send=kwargs.get('give_send')) connect.code_iter = code_iter @@ -806,3 +962,7 @@ def mocked_http_conn(*args, **kwargs): left_over_status = list(fake_conn.code_iter) if left_over_status: raise AssertionError('left over status %r' % left_over_status) + + +def make_timestamp_iter(): + return iter(Timestamp(t) for t in itertools.count(int(time.time()))) diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index 6c1c102b8..d81b565fc 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -141,7 +141,7 @@ cont_nodes = [{'device': 'sda1', @unit.patch_policies([StoragePolicy(0, 'zero', False, object_ring=unit.FakeRing()), StoragePolicy(1, 'one', True, - object_ring=unit.FakeRing())]) + object_ring=unit.FakeRing(replicas=4))]) class TestReaper(unittest.TestCase): def setUp(self): @@ -215,7 +215,7 @@ class TestReaper(unittest.TestCase): r.stats_objects_possibly_remaining = 0 r.myips = myips if fakelogger: - r.logger = FakeLogger() + r.logger = unit.debug_logger('test-reaper') return r def fake_reap_account(self, *args, **kwargs): @@ -287,7 +287,7 @@ class TestReaper(unittest.TestCase): policy.idx) for i, call_args in enumerate( fake_direct_delete.call_args_list): - cnode = cont_nodes[i] + cnode = cont_nodes[i % len(cont_nodes)] host = '%(ip)s:%(port)s' % cnode device = cnode['device'] headers = { @@ -297,11 +297,13 @@ class TestReaper(unittest.TestCase): 'X-Backend-Storage-Policy-Index': policy.idx } ring = r.get_object_ring(policy.idx) - expected = call(ring.devs[i], 0, 'a', 'c', 'o', + expected = call(dict(ring.devs[i], index=i), 0, + 'a', 'c', 'o', headers=headers, conn_timeout=0.5, response_timeout=10) self.assertEqual(call_args, expected) - self.assertEqual(r.stats_objects_deleted, 3) + self.assertEqual(r.stats_objects_deleted, + policy.object_ring.replicas) def test_reap_object_fail(self): r = self.init_reaper({}, fakelogger=True) @@ -312,7 +314,26 @@ class TestReaper(unittest.TestCase): self.fake_direct_delete_object): r.reap_object('a', 'c', 'partition', cont_nodes, 'o', policy.idx) - self.assertEqual(r.stats_objects_deleted, 1) + # IMHO, the stat handling in the node loop of reap object is + # over indented, but no one has complained, so I'm not inclined + # to move it. However it's worth noting we're currently keeping + # stats on deletes per *replica* - which is rather obvious from + # these tests, but this results is surprising because of some + # funny logic to *skip* increments on successful deletes of + # replicas until we have more successful responses than + # failures. This means that while the first replica doesn't + # increment deleted because of the failure, the second one + # *does* get successfully deleted, but *also does not* increment + # the counter (!?). + # + # In the three replica case this leaves only the last deleted + # object incrementing the counter - in the four replica case + # this leaves the last two. + # + # Basically this test will always result in: + # deleted == num_replicas - 2 + self.assertEqual(r.stats_objects_deleted, + policy.object_ring.replicas - 2) self.assertEqual(r.stats_objects_remaining, 1) self.assertEqual(r.stats_objects_possibly_remaining, 1) @@ -347,7 +368,7 @@ class TestReaper(unittest.TestCase): mocks['direct_get_container'].side_effect = fake_get_container r.reap_container('a', 'partition', acc_nodes, 'c') mock_calls = mocks['direct_delete_object'].call_args_list - self.assertEqual(3, len(mock_calls)) + self.assertEqual(policy.object_ring.replicas, len(mock_calls)) for call_args in mock_calls: _args, kwargs = call_args self.assertEqual(kwargs['headers'] @@ -355,7 +376,7 @@ class TestReaper(unittest.TestCase): policy.idx) self.assertEquals(mocks['direct_delete_container'].call_count, 3) - self.assertEqual(r.stats_objects_deleted, 3) + self.assertEqual(r.stats_objects_deleted, policy.object_ring.replicas) def test_reap_container_get_object_fail(self): r = self.init_reaper({}, fakelogger=True) @@ -373,7 +394,7 @@ class TestReaper(unittest.TestCase): self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') - self.assertEqual(r.logger.inc['return_codes.4'], 1) + self.assertEqual(r.logger.get_increment_counts()['return_codes.4'], 1) self.assertEqual(r.stats_containers_deleted, 1) def test_reap_container_partial_fail(self): @@ -392,7 +413,7 @@ class TestReaper(unittest.TestCase): self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') - self.assertEqual(r.logger.inc['return_codes.4'], 2) + self.assertEqual(r.logger.get_increment_counts()['return_codes.4'], 2) self.assertEqual(r.stats_containers_possibly_remaining, 1) def test_reap_container_full_fail(self): @@ -411,7 +432,7 @@ class TestReaper(unittest.TestCase): self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') - self.assertEqual(r.logger.inc['return_codes.4'], 3) + self.assertEqual(r.logger.get_increment_counts()['return_codes.4'], 3) self.assertEqual(r.stats_containers_remaining, 1) @patch('swift.account.reaper.Ring', @@ -436,8 +457,8 @@ class TestReaper(unittest.TestCase): mocks['direct_get_container'].side_effect = fake_get_container r.reap_container('a', 'partition', acc_nodes, 'c') - self.assertEqual(r.logger.msg, - 'ERROR: invalid storage policy index: 2') + self.assertEqual(r.logger.get_lines_for_level('error'), [ + 'ERROR: invalid storage policy index: 2']) def fake_reap_container(self, *args, **kwargs): self.called_amount += 1 @@ -462,13 +483,16 @@ class TestReaper(unittest.TestCase): nodes = r.get_account_ring().get_part_nodes() self.assertTrue(r.reap_account(broker, 'partition', nodes)) self.assertEqual(self.called_amount, 4) - self.assertEqual(r.logger.msg.find('Completed pass'), 0) - self.assertTrue(r.logger.msg.find('1 containers deleted')) - self.assertTrue(r.logger.msg.find('1 objects deleted')) - self.assertTrue(r.logger.msg.find('1 containers remaining')) - self.assertTrue(r.logger.msg.find('1 objects remaining')) - self.assertTrue(r.logger.msg.find('1 containers possibly remaining')) - self.assertTrue(r.logger.msg.find('1 objects possibly remaining')) + info_lines = r.logger.get_lines_for_level('info') + self.assertEqual(len(info_lines), 2) + start_line, stat_line = info_lines + self.assertEqual(start_line, 'Beginning pass on account a') + self.assertTrue(stat_line.find('1 containers deleted')) + self.assertTrue(stat_line.find('1 objects deleted')) + self.assertTrue(stat_line.find('1 containers remaining')) + self.assertTrue(stat_line.find('1 objects remaining')) + self.assertTrue(stat_line.find('1 containers possibly remaining')) + self.assertTrue(stat_line.find('1 objects possibly remaining')) def test_reap_account_no_container(self): broker = FakeAccountBroker(tuple()) @@ -482,7 +506,8 @@ class TestReaper(unittest.TestCase): with nested(*ctx): nodes = r.get_account_ring().get_part_nodes() self.assertTrue(r.reap_account(broker, 'partition', nodes)) - self.assertEqual(r.logger.msg.find('Completed pass'), 0) + self.assertTrue(r.logger.get_lines_for_level( + 'info')[-1].startswith('Completed pass')) self.assertEqual(self.called_amount, 0) def test_reap_device(self): diff --git a/test/unit/common/middleware/test_dlo.py b/test/unit/common/middleware/test_dlo.py index a292bc92b..16237eb1d 100644 --- a/test/unit/common/middleware/test_dlo.py +++ b/test/unit/common/middleware/test_dlo.py @@ -564,9 +564,10 @@ class TestDloGetManifest(DloTestCase): environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) self.assertEqual(status, "409 Conflict") - err_log = self.dlo.logger.log_dict['exception'][0][0][0] - self.assertTrue(err_log.startswith('ERROR: An error occurred ' - 'while retrieving segments')) + err_lines = self.dlo.logger.get_lines_for_level('error') + self.assertEqual(len(err_lines), 1) + self.assertTrue(err_lines[0].startswith( + 'ERROR: An error occurred while retrieving segments')) def test_error_fetching_second_segment(self): self.app.register( @@ -581,9 +582,10 @@ class TestDloGetManifest(DloTestCase): self.assertTrue(isinstance(exc, exceptions.SegmentError)) self.assertEqual(status, "200 OK") self.assertEqual(''.join(body), "aaaaa") # first segment made it out - err_log = self.dlo.logger.log_dict['exception'][0][0][0] - self.assertTrue(err_log.startswith('ERROR: An error occurred ' - 'while retrieving segments')) + err_lines = self.dlo.logger.get_lines_for_level('error') + self.assertEqual(len(err_lines), 1) + self.assertTrue(err_lines[0].startswith( + 'ERROR: An error occurred while retrieving segments')) def test_error_listing_container_first_listing_request(self): self.app.register( diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index f4bac155c..4160d91d4 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -1431,9 +1431,10 @@ class TestSloGetManifest(SloTestCase): self.assertEqual(status, '409 Conflict') self.assertEqual(self.app.call_count, 10) - err_log = self.slo.logger.log_dict['exception'][0][0][0] - self.assertTrue(err_log.startswith('ERROR: An error occurred ' - 'while retrieving segments')) + error_lines = self.slo.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + self.assertTrue(error_lines[0].startswith( + 'ERROR: An error occurred while retrieving segments')) def test_get_with_if_modified_since(self): # It's important not to pass the If-[Un]Modified-Since header to the @@ -1508,9 +1509,10 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) - err_log = self.slo.logger.log_dict['exception'][0][0][0] - self.assertTrue(err_log.startswith('ERROR: An error occurred ' - 'while retrieving segments')) + error_lines = self.slo.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + self.assertTrue(error_lines[0].startswith( + 'ERROR: An error occurred while retrieving segments')) def test_invalid_json_submanifest(self): self.app.register( @@ -1585,9 +1587,10 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) - err_log = self.slo.logger.log_dict['exception'][0][0][0] - self.assertTrue(err_log.startswith('ERROR: An error occurred ' - 'while retrieving segments')) + error_lines = self.slo.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + self.assertTrue(error_lines[0].startswith( + 'ERROR: An error occurred while retrieving segments')) def test_first_segment_mismatched_size(self): self.app.register('GET', '/v1/AUTH_test/gettest/manifest-badsize', @@ -1603,9 +1606,10 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) - err_log = self.slo.logger.log_dict['exception'][0][0][0] - self.assertTrue(err_log.startswith('ERROR: An error occurred ' - 'while retrieving segments')) + error_lines = self.slo.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + self.assertTrue(error_lines[0].startswith( + 'ERROR: An error occurred while retrieving segments')) def test_download_takes_too_long(self): the_time = [time.time()] @@ -1657,9 +1661,10 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) - err_log = self.slo.logger.log_dict['exception'][0][0][0] - self.assertTrue(err_log.startswith('ERROR: An error occurred ' - 'while retrieving segments')) + error_lines = self.slo.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + self.assertTrue(error_lines[0].startswith( + 'ERROR: An error occurred while retrieving segments')) class TestSloBulkLogger(unittest.TestCase): diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index fff715785..b97b60eee 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -363,63 +363,74 @@ class TestRing(TestRingBase): self.assertRaises(TypeError, self.ring.get_nodes) part, nodes = self.ring.get_nodes('a') self.assertEquals(part, 0) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) part, nodes = self.ring.get_nodes('a1') self.assertEquals(part, 0) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) part, nodes = self.ring.get_nodes('a4') self.assertEquals(part, 1) - self.assertEquals(nodes, [self.intended_devs[1], - self.intended_devs[4]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[1], + self.intended_devs[4]])]) part, nodes = self.ring.get_nodes('aa') self.assertEquals(part, 1) - self.assertEquals(nodes, [self.intended_devs[1], - self.intended_devs[4]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[1], + self.intended_devs[4]])]) part, nodes = self.ring.get_nodes('a', 'c1') self.assertEquals(part, 0) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) part, nodes = self.ring.get_nodes('a', 'c0') self.assertEquals(part, 3) - self.assertEquals(nodes, [self.intended_devs[1], - self.intended_devs[4]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[1], + self.intended_devs[4]])]) part, nodes = self.ring.get_nodes('a', 'c3') self.assertEquals(part, 2) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) part, nodes = self.ring.get_nodes('a', 'c2') - self.assertEquals(part, 2) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) part, nodes = self.ring.get_nodes('a', 'c', 'o1') self.assertEquals(part, 1) - self.assertEquals(nodes, [self.intended_devs[1], - self.intended_devs[4]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[1], + self.intended_devs[4]])]) part, nodes = self.ring.get_nodes('a', 'c', 'o5') self.assertEquals(part, 0) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) part, nodes = self.ring.get_nodes('a', 'c', 'o0') self.assertEquals(part, 0) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) part, nodes = self.ring.get_nodes('a', 'c', 'o2') self.assertEquals(part, 2) - self.assertEquals(nodes, [self.intended_devs[0], - self.intended_devs[3]]) + self.assertEquals(nodes, [dict(node, index=i) for i, node in + enumerate([self.intended_devs[0], + self.intended_devs[3]])]) def add_dev_to_ring(self, new_dev): self.ring.devs.append(new_dev) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 7ae9fb44a..61231d3f0 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -368,6 +368,11 @@ class TestConstraints(unittest.TestCase): self.assertTrue('X-Delete-At' in req.headers) self.assertEqual(req.headers['X-Delete-At'], expected) + def test_check_dir(self): + self.assertFalse(constraints.check_dir('', '')) + with mock.patch("os.path.isdir", MockTrue()): + self.assertTrue(constraints.check_dir('/srv', 'foo/bar')) + def test_check_mount(self): self.assertFalse(constraints.check_mount('', '')) with mock.patch("swift.common.utils.ismount", MockTrue()): diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index d4027261d..b7d680688 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -235,19 +235,20 @@ class TestInternalClient(unittest.TestCase): write_fake_ring(object_ring_path) with patch_policies([StoragePolicy(0, 'legacy', True)]): client = internal_client.InternalClient(conf_path, 'test', 1) - self.assertEqual(client.account_ring, client.app.app.app.account_ring) - self.assertEqual(client.account_ring.serialized_path, - account_ring_path) - self.assertEqual(client.container_ring, - client.app.app.app.container_ring) - self.assertEqual(client.container_ring.serialized_path, - container_ring_path) - object_ring = client.app.app.app.get_object_ring(0) - self.assertEqual(client.get_object_ring(0), - object_ring) - self.assertEqual(object_ring.serialized_path, - object_ring_path) - self.assertEquals(client.auto_create_account_prefix, '-') + self.assertEqual(client.account_ring, + client.app.app.app.account_ring) + self.assertEqual(client.account_ring.serialized_path, + account_ring_path) + self.assertEqual(client.container_ring, + client.app.app.app.container_ring) + self.assertEqual(client.container_ring.serialized_path, + container_ring_path) + object_ring = client.app.app.app.get_object_ring(0) + self.assertEqual(client.get_object_ring(0), + object_ring) + self.assertEqual(object_ring.serialized_path, + object_ring_path) + self.assertEquals(client.auto_create_account_prefix, '-') def test_init(self): class App(object): diff --git a/test/unit/common/test_request_helpers.py b/test/unit/common/test_request_helpers.py index c87a39979..d2dc02c48 100644 --- a/test/unit/common/test_request_helpers.py +++ b/test/unit/common/test_request_helpers.py @@ -16,10 +16,13 @@ """Tests for swift.common.request_helpers""" import unittest -from swift.common.swob import Request +from swift.common.swob import Request, HTTPException +from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY from swift.common.request_helpers import is_sys_meta, is_user_meta, \ is_sys_or_user_meta, strip_sys_meta_prefix, strip_user_meta_prefix, \ - remove_items, copy_header_subset + remove_items, copy_header_subset, get_name_and_placement + +from test.unit import patch_policies server_types = ['account', 'container', 'object'] @@ -81,3 +84,77 @@ class TestRequestHelpers(unittest.TestCase): self.assertEqual(to_req.headers['A'], 'b') self.assertFalse('c' in to_req.headers) self.assertFalse('C' in to_req.headers) + + @patch_policies(with_ec_default=True) + def test_get_name_and_placement_object_req(self): + path = '/device/part/account/container/object' + req = Request.blank(path, headers={ + 'X-Backend-Storage-Policy-Index': '0'}) + device, part, account, container, obj, policy = \ + get_name_and_placement(req, 5, 5, True) + self.assertEqual(device, 'device') + self.assertEqual(part, 'part') + self.assertEqual(account, 'account') + self.assertEqual(container, 'container') + self.assertEqual(obj, 'object') + self.assertEqual(policy, POLICIES[0]) + self.assertEqual(policy.policy_type, EC_POLICY) + + req.headers['X-Backend-Storage-Policy-Index'] = 1 + device, part, account, container, obj, policy = \ + get_name_and_placement(req, 5, 5, True) + self.assertEqual(device, 'device') + self.assertEqual(part, 'part') + self.assertEqual(account, 'account') + self.assertEqual(container, 'container') + self.assertEqual(obj, 'object') + self.assertEqual(policy, POLICIES[1]) + self.assertEqual(policy.policy_type, REPL_POLICY) + + req.headers['X-Backend-Storage-Policy-Index'] = 'foo' + try: + device, part, account, container, obj, policy = \ + get_name_and_placement(req, 5, 5, True) + except HTTPException as e: + self.assertEqual(e.status_int, 503) + self.assertEqual(str(e), '503 Service Unavailable') + self.assertEqual(e.body, "No policy with index foo") + else: + self.fail('get_name_and_placement did not raise error ' + 'for invalid storage policy index') + + @patch_policies(with_ec_default=True) + def test_get_name_and_placement_object_replication(self): + # yup, suffixes are sent '-'.joined in the path + path = '/device/part/012-345-678-9ab-cde' + req = Request.blank(path, headers={ + 'X-Backend-Storage-Policy-Index': '0'}) + device, partition, suffix_parts, policy = \ + get_name_and_placement(req, 2, 3, True) + self.assertEqual(device, 'device') + self.assertEqual(partition, 'part') + self.assertEqual(suffix_parts, '012-345-678-9ab-cde') + self.assertEqual(policy, POLICIES[0]) + self.assertEqual(policy.policy_type, EC_POLICY) + + path = '/device/part' + req = Request.blank(path, headers={ + 'X-Backend-Storage-Policy-Index': '1'}) + device, partition, suffix_parts, policy = \ + get_name_and_placement(req, 2, 3, True) + self.assertEqual(device, 'device') + self.assertEqual(partition, 'part') + self.assertEqual(suffix_parts, None) # false-y + self.assertEqual(policy, POLICIES[1]) + self.assertEqual(policy.policy_type, REPL_POLICY) + + path = '/device/part/' # with a trailing slash + req = Request.blank(path, headers={ + 'X-Backend-Storage-Policy-Index': '1'}) + device, partition, suffix_parts, policy = \ + get_name_and_placement(req, 2, 3, True) + self.assertEqual(device, 'device') + self.assertEqual(partition, 'part') + self.assertEqual(suffix_parts, '') # still false-y + self.assertEqual(policy, POLICIES[1]) + self.assertEqual(policy.policy_type, REPL_POLICY) diff --git a/test/unit/common/test_storage_policy.py b/test/unit/common/test_storage_policy.py index 21fed77ee..6406dc192 100644 --- a/test/unit/common/test_storage_policy.py +++ b/test/unit/common/test_storage_policy.py @@ -19,8 +19,23 @@ import mock from tempfile import NamedTemporaryFile from test.unit import patch_policies, FakeRing from swift.common.storage_policy import ( - StoragePolicy, StoragePolicyCollection, POLICIES, PolicyError, - parse_storage_policies, reload_storage_policies, get_policy_string) + StoragePolicyCollection, POLICIES, PolicyError, parse_storage_policies, + reload_storage_policies, get_policy_string, split_policy_string, + BaseStoragePolicy, StoragePolicy, ECStoragePolicy, REPL_POLICY, EC_POLICY, + VALID_EC_TYPES, DEFAULT_EC_OBJECT_SEGMENT_SIZE) +from swift.common.exceptions import RingValidationError + + +@BaseStoragePolicy.register('fake') +class FakeStoragePolicy(BaseStoragePolicy): + """ + Test StoragePolicy class - the only user at the moment is + test_validate_policies_type_invalid() + """ + def __init__(self, idx, name='', is_default=False, is_deprecated=False, + object_ring=None): + super(FakeStoragePolicy, self).__init__( + idx, name, is_default, is_deprecated, object_ring) class TestStoragePolicies(unittest.TestCase): @@ -31,15 +46,35 @@ class TestStoragePolicies(unittest.TestCase): conf.readfp(StringIO.StringIO(conf_str)) return conf - @patch_policies([StoragePolicy(0, 'zero', True), - StoragePolicy(1, 'one', False), - StoragePolicy(2, 'two', False), - StoragePolicy(3, 'three', False, is_deprecated=True)]) + def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs): + try: + f(*args, **kwargs) + except exc_class as err: + err_msg = str(err) + self.assert_(message in err_msg, 'Error message %r did not ' + 'have expected substring %r' % (err_msg, message)) + else: + self.fail('%r did not raise %s' % (message, exc_class.__name__)) + + def test_policy_baseclass_instantiate(self): + self.assertRaisesWithMessage(TypeError, + "Can't instantiate BaseStoragePolicy", + BaseStoragePolicy, 1, 'one') + + @patch_policies([ + StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one'), + StoragePolicy(2, 'two'), + StoragePolicy(3, 'three', is_deprecated=True), + ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=4), + ]) def test_swift_info(self): # the deprecated 'three' should not exist in expect expect = [{'default': True, 'name': 'zero'}, {'name': 'two'}, - {'name': 'one'}] + {'name': 'one'}, + {'name': 'ten'}] swift_info = POLICIES.get_policy_info() self.assertEquals(sorted(expect, key=lambda k: k['name']), sorted(swift_info, key=lambda k: k['name'])) @@ -48,10 +83,48 @@ class TestStoragePolicies(unittest.TestCase): def test_get_policy_string(self): self.assertEquals(get_policy_string('something', 0), 'something') self.assertEquals(get_policy_string('something', None), 'something') + self.assertEquals(get_policy_string('something', ''), 'something') self.assertEquals(get_policy_string('something', 1), 'something' + '-1') self.assertRaises(PolicyError, get_policy_string, 'something', 99) + @patch_policies + def test_split_policy_string(self): + expectations = { + 'something': ('something', POLICIES[0]), + 'something-1': ('something', POLICIES[1]), + 'tmp': ('tmp', POLICIES[0]), + 'objects': ('objects', POLICIES[0]), + 'tmp-1': ('tmp', POLICIES[1]), + 'objects-1': ('objects', POLICIES[1]), + 'objects-': PolicyError, + 'objects-0': PolicyError, + 'objects--1': ('objects-', POLICIES[1]), + 'objects-+1': PolicyError, + 'objects--': PolicyError, + 'objects-foo': PolicyError, + 'objects--bar': PolicyError, + 'objects-+bar': PolicyError, + # questionable, demonstrated as inverse of get_policy_string + 'objects+0': ('objects+0', POLICIES[0]), + '': ('', POLICIES[0]), + '0': ('0', POLICIES[0]), + '-1': ('', POLICIES[1]), + } + for policy_string, expected in expectations.items(): + if expected == PolicyError: + try: + invalid = split_policy_string(policy_string) + except PolicyError: + continue # good + else: + self.fail('The string %r returned %r ' + 'instead of raising a PolicyError' % + (policy_string, invalid)) + self.assertEqual(expected, split_policy_string(policy_string)) + # should be inverse of get_policy_string + self.assertEqual(policy_string, get_policy_string(*expected)) + def test_defaults(self): self.assertTrue(len(POLICIES) > 0) @@ -66,7 +139,9 @@ class TestStoragePolicies(unittest.TestCase): def test_storage_policy_repr(self): test_policies = [StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), - StoragePolicy(2, 'cee', False)] + StoragePolicy(2, 'cee', False), + ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=3)] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) @@ -75,6 +150,13 @@ class TestStoragePolicies(unittest.TestCase): self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) + if policy.policy_type == EC_POLICY: + self.assert_('ec_type=%s' % policy.ec_type in policy_repr) + self.assert_('ec_ndata=%s' % policy.ec_ndata in policy_repr) + self.assert_('ec_nparity=%s' % + policy.ec_nparity in policy_repr) + self.assert_('ec_segment_size=%s' % + policy.ec_segment_size in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) @@ -157,15 +239,16 @@ class TestStoragePolicies(unittest.TestCase): def test_validate_policy_params(self): StoragePolicy(0, 'name') # sanity # bogus indexes - self.assertRaises(PolicyError, StoragePolicy, 'x', 'name') - self.assertRaises(PolicyError, StoragePolicy, -1, 'name') + self.assertRaises(PolicyError, FakeStoragePolicy, 'x', 'name') + self.assertRaises(PolicyError, FakeStoragePolicy, -1, 'name') + # non-zero Policy-0 - self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy, - 1, 'policy-0') + self.assertRaisesWithMessage(PolicyError, 'reserved', + FakeStoragePolicy, 1, 'policy-0') # deprecate default self.assertRaisesWithMessage( PolicyError, 'Deprecated policy can not be default', - StoragePolicy, 1, 'Policy-1', is_default=True, + FakeStoragePolicy, 1, 'Policy-1', is_default=True, is_deprecated=True) # weird names names = ( @@ -178,7 +261,7 @@ class TestStoragePolicies(unittest.TestCase): ) for name in names: self.assertRaisesWithMessage(PolicyError, 'Invalid name', - StoragePolicy, 1, name) + FakeStoragePolicy, 1, name) def test_validate_policies_names(self): # duplicate names @@ -188,6 +271,40 @@ class TestStoragePolicies(unittest.TestCase): self.assertRaises(PolicyError, StoragePolicyCollection, test_policies) + def test_validate_policies_type_default(self): + # no type specified - make sure the policy is initialized to + # DEFAULT_POLICY_TYPE + test_policy = FakeStoragePolicy(0, 'zero', True) + self.assertEquals(test_policy.policy_type, 'fake') + + def test_validate_policies_type_invalid(self): + class BogusStoragePolicy(FakeStoragePolicy): + policy_type = 'bogus' + # unsupported policy type - initialization with FakeStoragePolicy + self.assertRaisesWithMessage(PolicyError, 'Invalid type', + BogusStoragePolicy, 1, 'one') + + def test_policies_type_attribute(self): + test_policies = [ + StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one'), + StoragePolicy(2, 'two'), + StoragePolicy(3, 'three', is_deprecated=True), + ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=3), + ] + policies = StoragePolicyCollection(test_policies) + self.assertEquals(policies.get_by_index(0).policy_type, + REPL_POLICY) + self.assertEquals(policies.get_by_index(1).policy_type, + REPL_POLICY) + self.assertEquals(policies.get_by_index(2).policy_type, + REPL_POLICY) + self.assertEquals(policies.get_by_index(3).policy_type, + REPL_POLICY) + self.assertEquals(policies.get_by_index(10).policy_type, + EC_POLICY) + def test_names_are_normalized(self): test_policies = [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'ZERO', False)] @@ -207,16 +324,6 @@ class TestStoragePolicies(unittest.TestCase): self.assertEqual(pol1, policies.get_by_name(name)) self.assertEqual(policies.get_by_name(name).name, 'One') - def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs): - try: - f(*args, **kwargs) - except exc_class as err: - err_msg = str(err) - self.assert_(message in err_msg, 'Error message %r did not ' - 'have expected substring %r' % (err_msg, message)) - else: - self.fail('%r did not raise %s' % (message, exc_class.__name__)) - def test_deprecated_default(self): bad_conf = self._conf(""" [storage-policy:1] @@ -395,6 +502,133 @@ class TestStoragePolicies(unittest.TestCase): self.assertRaisesWithMessage(PolicyError, 'Invalid name', parse_storage_policies, bad_conf) + # policy_type = erasure_coding + + # missing ec_type, ec_num_data_fragments and ec_num_parity_fragments + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + """) + + self.assertRaisesWithMessage(PolicyError, 'Missing ec_type', + parse_storage_policies, bad_conf) + + # missing ec_type, but other options valid... + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + ec_num_data_fragments = 10 + ec_num_parity_fragments = 4 + """) + + self.assertRaisesWithMessage(PolicyError, 'Missing ec_type', + parse_storage_policies, bad_conf) + + # ec_type specified, but invalid... + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + default = yes + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + ec_type = garbage_alg + ec_num_data_fragments = 10 + ec_num_parity_fragments = 4 + """) + + self.assertRaisesWithMessage(PolicyError, + 'Wrong ec_type garbage_alg for policy ' + 'ec10-4, should be one of "%s"' % + (', '.join(VALID_EC_TYPES)), + parse_storage_policies, bad_conf) + + # missing and invalid ec_num_parity_fragments + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + ec_type = jerasure_rs_vand + ec_num_data_fragments = 10 + """) + + self.assertRaisesWithMessage(PolicyError, + 'Invalid ec_num_parity_fragments', + parse_storage_policies, bad_conf) + + for num_parity in ('-4', '0', 'x'): + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + ec_type = jerasure_rs_vand + ec_num_data_fragments = 10 + ec_num_parity_fragments = %s + """ % num_parity) + + self.assertRaisesWithMessage(PolicyError, + 'Invalid ec_num_parity_fragments', + parse_storage_policies, bad_conf) + + # missing and invalid ec_num_data_fragments + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + ec_type = jerasure_rs_vand + ec_num_parity_fragments = 4 + """) + + self.assertRaisesWithMessage(PolicyError, + 'Invalid ec_num_data_fragments', + parse_storage_policies, bad_conf) + + for num_data in ('-10', '0', 'x'): + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + ec_type = jerasure_rs_vand + ec_num_data_fragments = %s + ec_num_parity_fragments = 4 + """ % num_data) + + self.assertRaisesWithMessage(PolicyError, + 'Invalid ec_num_data_fragments', + parse_storage_policies, bad_conf) + + # invalid ec_object_segment_size + for segment_size in ('-4', '0', 'x'): + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = ec10-4 + policy_type = erasure_coding + ec_object_segment_size = %s + ec_type = jerasure_rs_vand + ec_num_data_fragments = 10 + ec_num_parity_fragments = 4 + """ % segment_size) + + self.assertRaisesWithMessage(PolicyError, + 'Invalid ec_object_segment_size', + parse_storage_policies, bad_conf) + # Additional section added to ensure parser ignores other sections conf = self._conf(""" [some-other-section] @@ -430,6 +664,8 @@ class TestStoragePolicies(unittest.TestCase): self.assertEquals("zero", policies.get_by_index(None).name) self.assertEquals("zero", policies.get_by_index('').name) + self.assertEqual(policies.get_by_index(0), policies.legacy) + def test_reload_invalid_storage_policies(self): conf = self._conf(""" [storage-policy:0] @@ -512,18 +748,124 @@ class TestStoragePolicies(unittest.TestCase): for policy in POLICIES: self.assertEqual(POLICIES[int(policy)], policy) - def test_storage_policy_get_options(self): - policy = StoragePolicy(1, 'gold', True, False) - self.assertEqual({'name': 'gold', - 'default': True, - 'deprecated': False}, - policy.get_options()) - - policy = StoragePolicy(1, 'gold', False, True) - self.assertEqual({'name': 'gold', - 'default': False, - 'deprecated': True}, - policy.get_options()) + def test_quorum_size_replication(self): + expected_sizes = {1: 1, + 2: 2, + 3: 2, + 4: 3, + 5: 3} + for n, expected in expected_sizes.items(): + policy = StoragePolicy(0, 'zero', + object_ring=FakeRing(replicas=n)) + self.assertEqual(policy.quorum, expected) + + def test_quorum_size_erasure_coding(self): + test_ec_policies = [ + ECStoragePolicy(10, 'ec8-2', ec_type='jerasure_rs_vand', + ec_ndata=8, ec_nparity=2), + ECStoragePolicy(11, 'df10-6', ec_type='flat_xor_hd_4', + ec_ndata=10, ec_nparity=6), + ] + for ec_policy in test_ec_policies: + k = ec_policy.ec_ndata + expected_size = \ + k + ec_policy.pyeclib_driver.min_parity_fragments_needed() + self.assertEqual(expected_size, ec_policy.quorum) + + def test_validate_ring(self): + test_policies = [ + ECStoragePolicy(0, 'ec8-2', ec_type='jerasure_rs_vand', + ec_ndata=8, ec_nparity=2, + object_ring=FakeRing(replicas=8), + is_default=True), + ECStoragePolicy(1, 'ec10-4', ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=4, + object_ring=FakeRing(replicas=10)), + ECStoragePolicy(2, 'ec4-2', ec_type='jerasure_rs_vand', + ec_ndata=4, ec_nparity=2, + object_ring=FakeRing(replicas=7)), + ] + policies = StoragePolicyCollection(test_policies) + + for policy in policies: + msg = 'EC ring for policy %s needs to be configured with ' \ + 'exactly %d nodes.' % \ + (policy.name, policy.ec_ndata + policy.ec_nparity) + self.assertRaisesWithMessage( + RingValidationError, msg, + policy._validate_ring) + + def test_storage_policy_get_info(self): + test_policies = [ + StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_deprecated=True), + ECStoragePolicy(10, 'ten', + ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=3), + ECStoragePolicy(11, 'done', is_deprecated=True, + ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=3), + ] + policies = StoragePolicyCollection(test_policies) + expected = { + # default replication + (0, True): { + 'name': 'zero', + 'default': True, + 'deprecated': False, + 'policy_type': REPL_POLICY + }, + (0, False): { + 'name': 'zero', + 'default': True, + }, + # deprecated replication + (1, True): { + 'name': 'one', + 'default': False, + 'deprecated': True, + 'policy_type': REPL_POLICY + }, + (1, False): { + 'name': 'one', + 'deprecated': True, + }, + # enabled ec + (10, True): { + 'name': 'ten', + 'default': False, + 'deprecated': False, + 'policy_type': EC_POLICY, + 'ec_type': 'jerasure_rs_vand', + 'ec_num_data_fragments': 10, + 'ec_num_parity_fragments': 3, + 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, + }, + (10, False): { + 'name': 'ten', + }, + # deprecated ec + (11, True): { + 'name': 'done', + 'default': False, + 'deprecated': True, + 'policy_type': EC_POLICY, + 'ec_type': 'jerasure_rs_vand', + 'ec_num_data_fragments': 10, + 'ec_num_parity_fragments': 3, + 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, + }, + (11, False): { + 'name': 'done', + 'deprecated': True, + }, + } + self.maxDiff = None + for policy in policies: + expected_info = expected[(int(policy), True)] + self.assertEqual(policy.get_info(config=True), expected_info) + expected_info = expected[(int(policy), False)] + self.assertEqual(policy.get_info(config=False), expected_info) if __name__ == '__main__': diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index fffb33ecf..7015abb8e 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -1553,6 +1553,17 @@ class TestConditionalIfMatch(unittest.TestCase): self.assertEquals(resp.status_int, 200) self.assertEquals(body, 'hi') + def test_simple_conditional_etag_match(self): + # if etag matches, proceed as normal + req = swift.common.swob.Request.blank( + '/', headers={'If-Match': 'not-the-etag'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + resp._conditional_etag = 'not-the-etag' + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + def test_quoted_simple_match(self): # double quotes or not, doesn't matter req = swift.common.swob.Request.blank( @@ -1573,6 +1584,16 @@ class TestConditionalIfMatch(unittest.TestCase): self.assertEquals(resp.status_int, 412) self.assertEquals(body, '') + def test_simple_conditional_etag_no_match(self): + req = swift.common.swob.Request.blank( + '/', headers={'If-Match': 'the-etag'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + resp._conditional_etag = 'not-the-etag' + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 412) + self.assertEquals(body, '') + def test_match_star(self): # "*" means match anything; see RFC 2616 section 14.24 req = swift.common.swob.Request.blank( diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 1489501e5..22aa3db5e 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -2190,13 +2190,14 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertFalse(utils.streq_const_time('a', 'aaaaa')) self.assertFalse(utils.streq_const_time('ABC123', 'abc123')) - def test_quorum_size(self): + def test_replication_quorum_size(self): expected_sizes = {1: 1, 2: 2, 3: 2, 4: 3, 5: 3} - got_sizes = dict([(n, utils.quorum_size(n)) for n in expected_sizes]) + got_sizes = dict([(n, utils.quorum_size(n)) + for n in expected_sizes]) self.assertEqual(expected_sizes, got_sizes) def test_rsync_ip_ipv4_localhost(self): @@ -4593,6 +4594,22 @@ class TestLRUCache(unittest.TestCase): self.assertEqual(f.size(), 4) +class TestParseContentRange(unittest.TestCase): + def test_good(self): + start, end, total = utils.parse_content_range("bytes 100-200/300") + self.assertEqual(start, 100) + self.assertEqual(end, 200) + self.assertEqual(total, 300) + + def test_bad(self): + self.assertRaises(ValueError, utils.parse_content_range, + "100-300/500") + self.assertRaises(ValueError, utils.parse_content_range, + "bytes 100-200/aardvark") + self.assertRaises(ValueError, utils.parse_content_range, + "bytes bulbous-bouffant/4994801") + + class TestParseContentDisposition(unittest.TestCase): def test_basic_content_type(self): @@ -4622,7 +4639,8 @@ class TestIterMultipartMimeDocuments(unittest.TestCase): it.next() except MimeInvalid as err: exc = err - self.assertEquals(str(exc), 'invalid starting boundary') + self.assertTrue('invalid starting boundary' in str(exc)) + self.assertTrue('--unique' in str(exc)) def test_empty(self): it = utils.iter_multipart_mime_documents(StringIO('--unique'), diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index 67142decd..279eb8624 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -156,6 +156,27 @@ class TestWSGI(unittest.TestCase): logger.info('testing') self.assertEquals('proxy-server', log_name) + @with_tempdir + def test_loadapp_from_file(self, tempdir): + conf_path = os.path.join(tempdir, 'object-server.conf') + conf_body = """ + [app:main] + use = egg:swift#object + """ + contents = dedent(conf_body) + with open(conf_path, 'w') as f: + f.write(contents) + app = wsgi.loadapp(conf_path) + self.assertTrue(isinstance(app, obj_server.ObjectController)) + + def test_loadapp_from_string(self): + conf_body = """ + [app:main] + use = egg:swift#object + """ + app = wsgi.loadapp(wsgi.ConfigString(conf_body)) + self.assertTrue(isinstance(app, obj_server.ObjectController)) + def test_init_request_processor_from_conf_dir(self): config_dir = { 'proxy-server.conf.d/pipeline.conf': """ diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index aa5cebc28..8c6d89532 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -14,17 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re +import os import unittest from contextlib import nested +from textwrap import dedent import mock -from test.unit import FakeLogger +from test.unit import debug_logger from swift.container import sync from swift.common import utils +from swift.common.wsgi import ConfigString from swift.common.exceptions import ClientException from swift.common.storage_policy import StoragePolicy -from test.unit import patch_policies +import test +from test.unit import patch_policies, with_tempdir utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'endcap' @@ -71,6 +74,9 @@ class FakeContainerBroker(object): @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestContainerSync(unittest.TestCase): + def setUp(self): + self.logger = debug_logger('test-container-sync') + def test_FileLikeIter(self): # Retained test to show new FileLikeIter acts just like the removed # _Iter2FileLikeObject did. @@ -96,10 +102,55 @@ class TestContainerSync(unittest.TestCase): self.assertEquals(flo.read(), '') self.assertEquals(flo.read(2), '') - def test_init(self): + def assertLogMessage(self, msg_level, expected, skip=0): + for line in self.logger.get_lines_for_level(msg_level)[skip:]: + msg = 'expected %r not in %r' % (expected, line) + self.assertTrue(expected in line, msg) + + @with_tempdir + def test_init(self, tempdir): + ic_conf_path = os.path.join(tempdir, 'internal-client.conf') cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) + self.assertTrue(cs.container_ring is cring) + + # specified but not exists will not start + conf = {'internal_client_conf_path': ic_conf_path} + self.assertRaises(SystemExit, sync.ContainerSync, conf, + container_ring=cring, logger=self.logger) + + # not specified will use default conf + with mock.patch('swift.container.sync.InternalClient') as mock_ic: + cs = sync.ContainerSync({}, container_ring=cring, + logger=self.logger) + self.assertTrue(cs.container_ring is cring) + self.assertTrue(mock_ic.called) + conf_path, name, retry = mock_ic.call_args[0] + self.assertTrue(isinstance(conf_path, ConfigString)) + self.assertEquals(conf_path.contents.getvalue(), + dedent(sync.ic_conf_body)) + self.assertLogMessage('warning', 'internal_client_conf_path') + self.assertLogMessage('warning', 'internal-client.conf-sample') + + # correct + contents = dedent(sync.ic_conf_body) + with open(ic_conf_path, 'w') as f: + f.write(contents) + with mock.patch('swift.container.sync.InternalClient') as mock_ic: + cs = sync.ContainerSync(conf, container_ring=cring) self.assertTrue(cs.container_ring is cring) + self.assertTrue(mock_ic.called) + conf_path, name, retry = mock_ic.call_args[0] + self.assertEquals(conf_path, ic_conf_path) + + sample_conf_filename = os.path.join( + os.path.dirname(test.__file__), + '../etc/internal-client.conf-sample') + with open(sample_conf_filename) as sample_conf_file: + sample_conf = sample_conf_file.read() + self.assertEqual(contents, sample_conf) def test_run_forever(self): # This runs runs_forever with fakes to succeed for two loops, the first @@ -142,7 +193,9 @@ class TestContainerSync(unittest.TestCase): 'storage_policy_index': 0}) sync.time = fake_time sync.sleep = fake_sleep - cs = sync.ContainerSync({}, container_ring=FakeRing()) + + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=FakeRing()) sync.audit_location_generator = fake_audit_location_generator cs.run_forever(1, 2, a=3, b=4, verbose=True) except Exception as err: @@ -197,7 +250,9 @@ class TestContainerSync(unittest.TestCase): p, info={'account': 'a', 'container': 'c', 'storage_policy_index': 0}) sync.time = fake_time - cs = sync.ContainerSync({}, container_ring=FakeRing()) + + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=FakeRing()) sync.audit_location_generator = fake_audit_location_generator cs.run_once(1, 2, a=3, b=4, verbose=True) self.assertEquals(time_calls, [6]) @@ -218,12 +273,14 @@ class TestContainerSync(unittest.TestCase): def test_container_sync_not_db(self): cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) self.assertEquals(cs.container_failures, 0) def test_container_sync_missing_db(self): cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) @@ -231,7 +288,8 @@ class TestContainerSync(unittest.TestCase): # Db could be there due to handoff replication so test that we ignore # those. cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) orig_ContainerBroker = sync.ContainerBroker try: sync.ContainerBroker = lambda p: FakeContainerBroker( @@ -263,7 +321,8 @@ class TestContainerSync(unittest.TestCase): def test_container_sync_deleted(self): cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) orig_ContainerBroker = sync.ContainerBroker try: sync.ContainerBroker = lambda p: FakeContainerBroker( @@ -288,7 +347,8 @@ class TestContainerSync(unittest.TestCase): def test_container_sync_no_to_or_key(self): cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) orig_ContainerBroker = sync.ContainerBroker try: sync.ContainerBroker = lambda p: FakeContainerBroker( @@ -368,7 +428,8 @@ class TestContainerSync(unittest.TestCase): def test_container_stop_at(self): cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) orig_ContainerBroker = sync.ContainerBroker orig_time = sync.time try: @@ -411,7 +472,8 @@ class TestContainerSync(unittest.TestCase): def test_container_first_loop(self): cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring) def fake_hash_path(account, container, obj, raw_digest=False): # Ensures that no rows match for full syncing, ordinal is 0 and @@ -543,7 +605,9 @@ class TestContainerSync(unittest.TestCase): def test_container_second_loop(self): cring = FakeRing() - cs = sync.ContainerSync({}, container_ring=cring) + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=cring, + logger=self.logger) orig_ContainerBroker = sync.ContainerBroker orig_hash_path = sync.hash_path orig_delete_object = sync.delete_object @@ -649,7 +713,6 @@ class TestContainerSync(unittest.TestCase): hex = 'abcdef' sync.uuid = FakeUUID - fake_logger = FakeLogger() def fake_delete_object(path, name=None, headers=None, proxy=None, logger=None, timeout=None): @@ -665,12 +728,14 @@ class TestContainerSync(unittest.TestCase): headers, {'x-container-sync-key': 'key', 'x-timestamp': '1.2'}) self.assertEquals(proxy, 'http://proxy') - self.assertEqual(logger, fake_logger) self.assertEqual(timeout, 5.0) + self.assertEqual(logger, self.logger) sync.delete_object = fake_delete_object - cs = sync.ContainerSync({}, container_ring=FakeRing()) - cs.logger = fake_logger + + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=FakeRing(), + logger=self.logger) cs.http_proxies = ['http://proxy'] # Success self.assertTrue(cs.container_sync_row( @@ -749,7 +814,6 @@ class TestContainerSync(unittest.TestCase): orig_uuid = sync.uuid orig_shuffle = sync.shuffle orig_put_object = sync.put_object - orig_direct_get_object = sync.direct_get_object try: class FakeUUID(object): class uuid4(object): @@ -757,7 +821,6 @@ class TestContainerSync(unittest.TestCase): sync.uuid = FakeUUID sync.shuffle = lambda x: x - fake_logger = FakeLogger() def fake_put_object(sync_to, name=None, headers=None, contents=None, proxy=None, logger=None, @@ -781,24 +844,25 @@ class TestContainerSync(unittest.TestCase): 'content-type': 'text/plain'}) self.assertEquals(contents.read(), 'contents') self.assertEquals(proxy, 'http://proxy') - self.assertEqual(logger, fake_logger) self.assertEqual(timeout, 5.0) + self.assertEqual(logger, self.logger) sync.put_object = fake_put_object - cs = sync.ContainerSync({}, container_ring=FakeRing()) - cs.logger = fake_logger + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync({}, container_ring=FakeRing(), + logger=self.logger) cs.http_proxies = ['http://proxy'] - def fake_direct_get_object(node, part, account, container, obj, - headers, resp_chunk_size=1): - self.assertEquals(headers['X-Backend-Storage-Policy-Index'], - '0') - return ({'other-header': 'other header value', - 'etag': '"etagvalue"', 'x-timestamp': '1.2', - 'content-type': 'text/plain; swift_bytes=123'}, + def fake_get_object(acct, con, obj, headers, acceptable_statuses): + self.assertEqual(headers['X-Backend-Storage-Policy-Index'], + '0') + return (200, {'other-header': 'other header value', + 'etag': '"etagvalue"', 'x-timestamp': '1.2', + 'content-type': 'text/plain; swift_bytes=123'}, iter('contents')) - sync.direct_get_object = fake_direct_get_object + + cs.swift.get_object = fake_get_object # Success as everything says it worked self.assertTrue(cs.container_sync_row( {'deleted': False, @@ -809,19 +873,19 @@ class TestContainerSync(unittest.TestCase): realm, realm_key)) self.assertEquals(cs.container_puts, 1) - def fake_direct_get_object(node, part, account, container, obj, - headers, resp_chunk_size=1): + def fake_get_object(acct, con, obj, headers, acceptable_statuses): + self.assertEquals(headers['X-Newest'], True) self.assertEquals(headers['X-Backend-Storage-Policy-Index'], '0') - return ({'date': 'date value', - 'last-modified': 'last modified value', - 'x-timestamp': '1.2', - 'other-header': 'other header value', - 'etag': '"etagvalue"', - 'content-type': 'text/plain; swift_bytes=123'}, + return (200, {'date': 'date value', + 'last-modified': 'last modified value', + 'x-timestamp': '1.2', + 'other-header': 'other header value', + 'etag': '"etagvalue"', + 'content-type': 'text/plain; swift_bytes=123'}, iter('contents')) - sync.direct_get_object = fake_direct_get_object + cs.swift.get_object = fake_get_object # Success as everything says it worked, also checks 'date' and # 'last-modified' headers are removed and that 'etag' header is # stripped of double quotes. @@ -836,14 +900,14 @@ class TestContainerSync(unittest.TestCase): exc = [] - def fake_direct_get_object(node, part, account, container, obj, - headers, resp_chunk_size=1): + def fake_get_object(acct, con, obj, headers, acceptable_statuses): + self.assertEquals(headers['X-Newest'], True) self.assertEquals(headers['X-Backend-Storage-Policy-Index'], '0') exc.append(Exception('test exception')) raise exc[-1] - sync.direct_get_object = fake_direct_get_object + cs.swift.get_object = fake_get_object # Fail due to completely unexpected exception self.assertFalse(cs.container_sync_row( {'deleted': False, @@ -853,22 +917,20 @@ class TestContainerSync(unittest.TestCase): {'account': 'a', 'container': 'c', 'storage_policy_index': 0}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) - self.assertEquals(len(exc), 3) + self.assertEquals(len(exc), 1) self.assertEquals(str(exc[-1]), 'test exception') exc = [] - def fake_direct_get_object(node, part, account, container, obj, - headers, resp_chunk_size=1): + def fake_get_object(acct, con, obj, headers, acceptable_statuses): + self.assertEquals(headers['X-Newest'], True) self.assertEquals(headers['X-Backend-Storage-Policy-Index'], '0') - if len(exc) == 0: - exc.append(Exception('test other exception')) - else: - exc.append(ClientException('test client exception')) + + exc.append(ClientException('test client exception')) raise exc[-1] - sync.direct_get_object = fake_direct_get_object + cs.swift.get_object = fake_get_object # Fail due to all direct_get_object calls failing self.assertFalse(cs.container_sync_row( {'deleted': False, @@ -878,25 +940,22 @@ class TestContainerSync(unittest.TestCase): {'account': 'a', 'container': 'c', 'storage_policy_index': 0}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) - self.assertEquals(len(exc), 3) - self.assertEquals(str(exc[-3]), 'test other exception') - self.assertEquals(str(exc[-2]), 'test client exception') + self.assertEquals(len(exc), 1) self.assertEquals(str(exc[-1]), 'test client exception') - def fake_direct_get_object(node, part, account, container, obj, - headers, resp_chunk_size=1): + def fake_get_object(acct, con, obj, headers, acceptable_statuses): + self.assertEquals(headers['X-Newest'], True) self.assertEquals(headers['X-Backend-Storage-Policy-Index'], '0') - return ({'other-header': 'other header value', - 'x-timestamp': '1.2', 'etag': '"etagvalue"'}, + return (200, {'other-header': 'other header value', + 'x-timestamp': '1.2', 'etag': '"etagvalue"'}, iter('contents')) def fake_put_object(*args, **kwargs): raise ClientException('test client exception', http_status=401) - sync.direct_get_object = fake_direct_get_object + cs.swift.get_object = fake_get_object sync.put_object = fake_put_object - cs.logger = FakeLogger() # Fail due to 401 self.assertFalse(cs.container_sync_row( {'deleted': False, @@ -906,15 +965,13 @@ class TestContainerSync(unittest.TestCase): {'account': 'a', 'container': 'c', 'storage_policy_index': 0}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) - self.assert_(re.match('Unauth ', - cs.logger.log_dict['info'][0][0][0])) + self.assertLogMessage('info', 'Unauth') def fake_put_object(*args, **kwargs): raise ClientException('test client exception', http_status=404) sync.put_object = fake_put_object # Fail due to 404 - cs.logger = FakeLogger() self.assertFalse(cs.container_sync_row( {'deleted': False, 'name': 'object', @@ -923,8 +980,7 @@ class TestContainerSync(unittest.TestCase): {'account': 'a', 'container': 'c', 'storage_policy_index': 0}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) - self.assert_(re.match('Not found ', - cs.logger.log_dict['info'][0][0][0])) + self.assertLogMessage('info', 'Not found', 1) def fake_put_object(*args, **kwargs): raise ClientException('test client exception', http_status=503) @@ -939,29 +995,32 @@ class TestContainerSync(unittest.TestCase): {'account': 'a', 'container': 'c', 'storage_policy_index': 0}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) - self.assertTrue( - cs.logger.log_dict['exception'][0][0][0].startswith( - 'ERROR Syncing ')) + self.assertLogMessage('error', 'ERROR Syncing') finally: sync.uuid = orig_uuid sync.shuffle = orig_shuffle sync.put_object = orig_put_object - sync.direct_get_object = orig_direct_get_object def test_select_http_proxy_None(self): - cs = sync.ContainerSync( - {'sync_proxy': ''}, container_ring=FakeRing()) + + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync( + {'sync_proxy': ''}, container_ring=FakeRing()) self.assertEqual(cs.select_http_proxy(), None) def test_select_http_proxy_one(self): - cs = sync.ContainerSync( - {'sync_proxy': 'http://one'}, container_ring=FakeRing()) + + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync( + {'sync_proxy': 'http://one'}, container_ring=FakeRing()) self.assertEqual(cs.select_http_proxy(), 'http://one') def test_select_http_proxy_multiple(self): - cs = sync.ContainerSync( - {'sync_proxy': 'http://one,http://two,http://three'}, - container_ring=FakeRing()) + + with mock.patch('swift.container.sync.InternalClient'): + cs = sync.ContainerSync( + {'sync_proxy': 'http://one,http://two,http://three'}, + container_ring=FakeRing()) self.assertEqual( set(cs.http_proxies), set(['http://one', 'http://two', 'http://three'])) diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index e8f8a2b16..3cfcb4757 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -28,7 +28,7 @@ from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash, \ get_data_dir, DiskFileManager, AuditLocation from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ storage_directory -from swift.common.storage_policy import StoragePolicy +from swift.common.storage_policy import StoragePolicy, POLICIES _mocked_policies = [StoragePolicy(0, 'zero', False), @@ -48,12 +48,16 @@ class TestAuditor(unittest.TestCase): os.mkdir(os.path.join(self.devices, 'sdb')) # policy 0 - self.objects = os.path.join(self.devices, 'sda', get_data_dir(0)) - self.objects_2 = os.path.join(self.devices, 'sdb', get_data_dir(0)) + self.objects = os.path.join(self.devices, 'sda', + get_data_dir(POLICIES[0])) + self.objects_2 = os.path.join(self.devices, 'sdb', + get_data_dir(POLICIES[0])) os.mkdir(self.objects) # policy 1 - self.objects_p1 = os.path.join(self.devices, 'sda', get_data_dir(1)) - self.objects_2_p1 = os.path.join(self.devices, 'sdb', get_data_dir(1)) + self.objects_p1 = os.path.join(self.devices, 'sda', + get_data_dir(POLICIES[1])) + self.objects_2_p1 = os.path.join(self.devices, 'sdb', + get_data_dir(POLICIES[1])) os.mkdir(self.objects_p1) self.parts = self.parts_p1 = {} @@ -70,9 +74,10 @@ class TestAuditor(unittest.TestCase): self.df_mgr = DiskFileManager(self.conf, self.logger) # diskfiles for policy 0, 1 - self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o', 0) + self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o', + policy=POLICIES[0]) self.disk_file_p1 = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', - 'o', 1) + 'o', policy=POLICIES[1]) def tearDown(self): rmtree(os.path.dirname(self.testdir), ignore_errors=1) @@ -125,13 +130,15 @@ class TestAuditor(unittest.TestCase): pre_quarantines = auditor_worker.quarantines auditor_worker.object_audit( - AuditLocation(disk_file._datadir, 'sda', '0')) + AuditLocation(disk_file._datadir, 'sda', '0', + policy=POLICIES.legacy)) self.assertEquals(auditor_worker.quarantines, pre_quarantines) os.write(writer._fd, 'extra_data') auditor_worker.object_audit( - AuditLocation(disk_file._datadir, 'sda', '0')) + AuditLocation(disk_file._datadir, 'sda', '0', + policy=POLICIES.legacy)) self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) run_tests(self.disk_file) @@ -156,10 +163,12 @@ class TestAuditor(unittest.TestCase): pre_quarantines = auditor_worker.quarantines # remake so it will have metadata - self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o') + self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o', + policy=POLICIES.legacy) auditor_worker.object_audit( - AuditLocation(self.disk_file._datadir, 'sda', '0')) + AuditLocation(self.disk_file._datadir, 'sda', '0', + policy=POLICIES.legacy)) self.assertEquals(auditor_worker.quarantines, pre_quarantines) etag = md5() etag.update('1' + '0' * 1023) @@ -171,7 +180,8 @@ class TestAuditor(unittest.TestCase): writer.put(metadata) auditor_worker.object_audit( - AuditLocation(self.disk_file._datadir, 'sda', '0')) + AuditLocation(self.disk_file._datadir, 'sda', '0', + policy=POLICIES.legacy)) self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_no_meta(self): @@ -186,7 +196,8 @@ class TestAuditor(unittest.TestCase): self.rcache, self.devices) pre_quarantines = auditor_worker.quarantines auditor_worker.object_audit( - AuditLocation(self.disk_file._datadir, 'sda', '0')) + AuditLocation(self.disk_file._datadir, 'sda', '0', + policy=POLICIES.legacy)) self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_will_not_swallow_errors_in_tests(self): @@ -203,7 +214,8 @@ class TestAuditor(unittest.TestCase): with mock.patch.object(DiskFileManager, 'get_diskfile_from_audit_location', blowup): self.assertRaises(NameError, auditor_worker.object_audit, - AuditLocation(os.path.dirname(path), 'sda', '0')) + AuditLocation(os.path.dirname(path), 'sda', '0', + policy=POLICIES.legacy)) def test_failsafe_object_audit_will_swallow_errors_in_tests(self): timestamp = str(normalize_timestamp(time.time())) @@ -216,9 +228,11 @@ class TestAuditor(unittest.TestCase): def blowup(*args): raise NameError('tpyo') - with mock.patch('swift.obj.diskfile.DiskFile', blowup): + with mock.patch('swift.obj.diskfile.DiskFileManager.diskfile_cls', + blowup): auditor_worker.failsafe_object_audit( - AuditLocation(os.path.dirname(path), 'sda', '0')) + AuditLocation(os.path.dirname(path), 'sda', '0', + policy=POLICIES.legacy)) self.assertEquals(auditor_worker.errors, 1) def test_generic_exception_handling(self): @@ -240,7 +254,8 @@ class TestAuditor(unittest.TestCase): 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) - with mock.patch('swift.obj.diskfile.DiskFile', lambda *_: 1 / 0): + with mock.patch('swift.obj.diskfile.DiskFileManager.diskfile_cls', + lambda *_: 1 / 0): auditor_worker.audit_all_objects() self.assertEquals(auditor_worker.errors, pre_errors + 1) @@ -368,7 +383,8 @@ class TestAuditor(unittest.TestCase): } writer.put(metadata) auditor_worker.audit_all_objects() - self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'ob') + self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'ob', + policy=POLICIES.legacy) data = '1' * 10 etag = md5() with self.disk_file.create() as writer: @@ -424,7 +440,7 @@ class TestAuditor(unittest.TestCase): name_hash = hash_path('a', 'c', 'o') dir_path = os.path.join( self.devices, 'sda', - storage_directory(get_data_dir(0), '0', name_hash)) + storage_directory(get_data_dir(POLICIES[0]), '0', name_hash)) ts_file_path = os.path.join(dir_path, '99999.ts') if not os.path.exists(dir_path): mkdirs(dir_path) @@ -474,9 +490,8 @@ class TestAuditor(unittest.TestCase): DiskFile._quarantine(self, data_file, msg) self.setup_bad_zero_byte() - was_df = auditor.diskfile.DiskFile - try: - auditor.diskfile.DiskFile = FakeFile + with mock.patch('swift.obj.diskfile.DiskFileManager.diskfile_cls', + FakeFile): kwargs = {'mode': 'once'} kwargs['zero_byte_fps'] = 50 self.auditor.run_audit(**kwargs) @@ -484,8 +499,6 @@ class TestAuditor(unittest.TestCase): 'sda', 'quarantined', 'objects') self.assertTrue(os.path.isdir(quarantine_path)) self.assertTrue(rat[0]) - finally: - auditor.diskfile.DiskFile = was_df @mock.patch.object(auditor.ObjectAuditor, 'run_audit') @mock.patch('os.fork', return_value=0) diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index cc6747555..2ccf3b136 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -19,6 +19,7 @@ import cPickle as pickle import os import errno +import itertools import mock import unittest import email @@ -26,6 +27,8 @@ import tempfile import uuid import xattr import re +from collections import defaultdict +from random import shuffle, randint from shutil import rmtree from time import time from tempfile import mkdtemp @@ -35,7 +38,7 @@ from gzip import GzipFile from eventlet import hubs, timeout, tpool from test.unit import (FakeLogger, mock as unit_mock, temptree, - patch_policies, debug_logger) + patch_policies, debug_logger, EMPTY_ETAG) from nose import SkipTest from swift.obj import diskfile @@ -45,32 +48,61 @@ from swift.common import ring from swift.common.splice import splice from swift.common.exceptions import DiskFileNotExist, DiskFileQuarantined, \ DiskFileDeviceUnavailable, DiskFileDeleted, DiskFileNotOpen, \ - DiskFileError, ReplicationLockTimeout, PathNotDir, DiskFileCollision, \ + DiskFileError, ReplicationLockTimeout, DiskFileCollision, \ DiskFileExpired, SwiftException, DiskFileNoSpace, DiskFileXattrNotSupported -from swift.common.storage_policy import POLICIES, get_policy_string -from functools import partial - - -get_data_dir = partial(get_policy_string, diskfile.DATADIR_BASE) -get_tmp_dir = partial(get_policy_string, diskfile.TMP_BASE) - - -def _create_test_ring(path): - testgz = os.path.join(path, 'object.ring.gz') +from swift.common.storage_policy import ( + POLICIES, get_policy_string, StoragePolicy, ECStoragePolicy, + BaseStoragePolicy, REPL_POLICY, EC_POLICY) + + +test_policies = [ + StoragePolicy(0, name='zero', is_default=True), + ECStoragePolicy(1, name='one', is_default=False, + ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=4), +] + + +def find_paths_with_matching_suffixes(needed_matches=2, needed_suffixes=3): + paths = defaultdict(list) + while True: + path = ('a', 'c', uuid.uuid4().hex) + hash_ = hash_path(*path) + suffix = hash_[-3:] + paths[suffix].append(path) + if len(paths) < needed_suffixes: + # in the extreamly unlikely situation where you land the matches + # you need before you get the total suffixes you need - it's + # simpler to just ignore this suffix for now + continue + if len(paths[suffix]) >= needed_matches: + break + return paths, suffix + + +def _create_test_ring(path, policy): + ring_name = get_policy_string('object', policy) + testgz = os.path.join(path, ring_name + '.ring.gz') intended_replica2part2dev_id = [ [0, 1, 2, 3, 4, 5, 6], [1, 2, 3, 0, 5, 6, 4], [2, 3, 0, 1, 6, 4, 5]] intended_devs = [ - {'id': 0, 'device': 'sda', 'zone': 0, 'ip': '127.0.0.0', 'port': 6000}, - {'id': 1, 'device': 'sda', 'zone': 1, 'ip': '127.0.0.1', 'port': 6000}, - {'id': 2, 'device': 'sda', 'zone': 2, 'ip': '127.0.0.2', 'port': 6000}, - {'id': 3, 'device': 'sda', 'zone': 4, 'ip': '127.0.0.3', 'port': 6000}, - {'id': 4, 'device': 'sda', 'zone': 5, 'ip': '127.0.0.4', 'port': 6000}, - {'id': 5, 'device': 'sda', 'zone': 6, + {'id': 0, 'device': 'sda1', 'zone': 0, 'ip': '127.0.0.0', + 'port': 6000}, + {'id': 1, 'device': 'sda1', 'zone': 1, 'ip': '127.0.0.1', + 'port': 6000}, + {'id': 2, 'device': 'sda1', 'zone': 2, 'ip': '127.0.0.2', + 'port': 6000}, + {'id': 3, 'device': 'sda1', 'zone': 4, 'ip': '127.0.0.3', + 'port': 6000}, + {'id': 4, 'device': 'sda1', 'zone': 5, 'ip': '127.0.0.4', + 'port': 6000}, + {'id': 5, 'device': 'sda1', 'zone': 6, 'ip': 'fe80::202:b3ff:fe1e:8329', 'port': 6000}, - {'id': 6, 'device': 'sda', 'zone': 7, - 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334', 'port': 6000}] + {'id': 6, 'device': 'sda1', 'zone': 7, + 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334', + 'port': 6000}] intended_part_shift = 30 intended_reload_time = 15 with closing(GzipFile(testgz, 'wb')) as f: @@ -78,7 +110,7 @@ def _create_test_ring(path): ring.RingData(intended_replica2part2dev_id, intended_devs, intended_part_shift), f) - return ring.Ring(path, ring_name='object', + return ring.Ring(path, ring_name=ring_name, reload_time=intended_reload_time) @@ -88,13 +120,13 @@ class TestDiskFileModuleMethods(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = '' - # Setup a test ring (stolen from common/test_ring.py) + # Setup a test ring per policy (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) os.mkdir(self.devices) - self.existing_device = 'sda' + self.existing_device = 'sda1' os.mkdir(os.path.join(self.devices, self.existing_device)) self.objects = os.path.join(self.devices, self.existing_device, 'objects') @@ -103,7 +135,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): for part in ['0', '1', '2', '3']: self.parts[part] = os.path.join(self.objects, part) os.mkdir(os.path.join(self.objects, part)) - self.ring = _create_test_ring(self.testdir) + self.ring = _create_test_ring(self.testdir, POLICIES.legacy) self.conf = dict( swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1') @@ -112,59 +144,58 @@ class TestDiskFileModuleMethods(unittest.TestCase): def tearDown(self): rmtree(self.testdir, ignore_errors=1) - def _create_diskfile(self, policy_idx=0): + def _create_diskfile(self, policy): return self.df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', 'o', - policy_idx) + policy=policy) - def test_extract_policy_index(self): + def test_extract_policy(self): # good path names pn = 'objects/0/606/1984527ed7ef6247c78606/1401379842.14643.data' - self.assertEqual(diskfile.extract_policy_index(pn), 0) + self.assertEqual(diskfile.extract_policy(pn), POLICIES[0]) pn = 'objects-1/0/606/198452b6ef6247c78606/1401379842.14643.data' - self.assertEqual(diskfile.extract_policy_index(pn), 1) + self.assertEqual(diskfile.extract_policy(pn), POLICIES[1]) + + # leading slash + pn = '/objects/0/606/1984527ed7ef6247c78606/1401379842.14643.data' + self.assertEqual(diskfile.extract_policy(pn), POLICIES[0]) + pn = '/objects-1/0/606/198452b6ef6247c78606/1401379842.14643.data' + self.assertEqual(diskfile.extract_policy(pn), POLICIES[1]) + + # full paths good_path = '/srv/node/sda1/objects-1/1/abc/def/1234.data' - self.assertEquals(1, diskfile.extract_policy_index(good_path)) + self.assertEqual(diskfile.extract_policy(good_path), POLICIES[1]) good_path = '/srv/node/sda1/objects/1/abc/def/1234.data' - self.assertEquals(0, diskfile.extract_policy_index(good_path)) + self.assertEqual(diskfile.extract_policy(good_path), POLICIES[0]) - # short paths still ok + # short paths path = '/srv/node/sda1/objects/1/1234.data' - self.assertEqual(diskfile.extract_policy_index(path), 0) + self.assertEqual(diskfile.extract_policy(path), POLICIES[0]) path = '/srv/node/sda1/objects-1/1/1234.data' - self.assertEqual(diskfile.extract_policy_index(path), 1) - - # leading slash, just in case - pn = '/objects/0/606/1984527ed7ef6247c78606/1401379842.14643.data' - self.assertEqual(diskfile.extract_policy_index(pn), 0) - pn = '/objects-1/0/606/198452b6ef6247c78606/1401379842.14643.data' - self.assertEqual(diskfile.extract_policy_index(pn), 1) + self.assertEqual(diskfile.extract_policy(path), POLICIES[1]) - # bad policy index + # well formatted but, unknown policy index pn = 'objects-2/0/606/198427efcff042c78606/1401379842.14643.data' - self.assertEqual(diskfile.extract_policy_index(pn), 0) - bad_path = '/srv/node/sda1/objects-t/1/abc/def/1234.data' - self.assertRaises(ValueError, - diskfile.extract_policy_index, bad_path) + self.assertEqual(diskfile.extract_policy(pn), None) - # malformed path (no objects dir or nothing at all) + # malformed path + self.assertEqual(diskfile.extract_policy(''), None) + bad_path = '/srv/node/sda1/objects-t/1/abc/def/1234.data' + self.assertEqual(diskfile.extract_policy(bad_path), None) pn = 'XXXX/0/606/1984527ed42b6ef6247c78606/1401379842.14643.data' - self.assertEqual(diskfile.extract_policy_index(pn), 0) - self.assertEqual(diskfile.extract_policy_index(''), 0) - - # no datadir base in path + self.assertEqual(diskfile.extract_policy(pn), None) bad_path = '/srv/node/sda1/foo-1/1/abc/def/1234.data' - self.assertEqual(diskfile.extract_policy_index(bad_path), 0) + self.assertEqual(diskfile.extract_policy(bad_path), None) bad_path = '/srv/node/sda1/obj1/1/abc/def/1234.data' - self.assertEqual(diskfile.extract_policy_index(bad_path), 0) + self.assertEqual(diskfile.extract_policy(bad_path), None) def test_quarantine_renamer(self): for policy in POLICIES: # we use this for convenience, not really about a diskfile layout - df = self._create_diskfile(policy_idx=policy.idx) + df = self._create_diskfile(policy=policy) mkdirs(df._datadir) exp_dir = os.path.join(self.devices, 'quarantined', - get_data_dir(policy.idx), + diskfile.get_data_dir(policy), os.path.basename(df._datadir)) qbit = os.path.join(df._datadir, 'qbit') with open(qbit, 'w') as f: @@ -174,38 +205,28 @@ class TestDiskFileModuleMethods(unittest.TestCase): self.assertRaises(OSError, diskfile.quarantine_renamer, self.devices, qbit) - def test_hash_suffix_enoent(self): - self.assertRaises(PathNotDir, diskfile.hash_suffix, - os.path.join(self.testdir, "doesnotexist"), 101) - - def test_hash_suffix_oserror(self): - mocked_os_listdir = mock.Mock( - side_effect=OSError(errno.EACCES, os.strerror(errno.EACCES))) - with mock.patch("os.listdir", mocked_os_listdir): - self.assertRaises(OSError, diskfile.hash_suffix, - os.path.join(self.testdir, "doesnotexist"), 101) - def test_get_data_dir(self): - self.assertEquals(diskfile.get_data_dir(0), diskfile.DATADIR_BASE) - self.assertEquals(diskfile.get_data_dir(1), + self.assertEquals(diskfile.get_data_dir(POLICIES[0]), + diskfile.DATADIR_BASE) + self.assertEquals(diskfile.get_data_dir(POLICIES[1]), diskfile.DATADIR_BASE + "-1") self.assertRaises(ValueError, diskfile.get_data_dir, 'junk') self.assertRaises(ValueError, diskfile.get_data_dir, 99) def test_get_async_dir(self): - self.assertEquals(diskfile.get_async_dir(0), + self.assertEquals(diskfile.get_async_dir(POLICIES[0]), diskfile.ASYNCDIR_BASE) - self.assertEquals(diskfile.get_async_dir(1), + self.assertEquals(diskfile.get_async_dir(POLICIES[1]), diskfile.ASYNCDIR_BASE + "-1") self.assertRaises(ValueError, diskfile.get_async_dir, 'junk') self.assertRaises(ValueError, diskfile.get_async_dir, 99) def test_get_tmp_dir(self): - self.assertEquals(diskfile.get_tmp_dir(0), + self.assertEquals(diskfile.get_tmp_dir(POLICIES[0]), diskfile.TMP_BASE) - self.assertEquals(diskfile.get_tmp_dir(1), + self.assertEquals(diskfile.get_tmp_dir(POLICIES[1]), diskfile.TMP_BASE + "-1") self.assertRaises(ValueError, diskfile.get_tmp_dir, 'junk') @@ -221,7 +242,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): self.devices, self.existing_device, tmp_part) self.assertFalse(os.path.isdir(tmp_path)) pickle_args = (self.existing_device, 'a', 'c', 'o', - 'data', 0.0, int(policy)) + 'data', 0.0, policy) # async updates don't create their tmpdir on their own self.assertRaises(OSError, self.df_mgr.pickle_async_update, *pickle_args) @@ -231,438 +252,6 @@ class TestDiskFileModuleMethods(unittest.TestCase): # check tempdir self.assertTrue(os.path.isdir(tmp_path)) - def test_hash_suffix_hash_dir_is_file_quarantine(self): - df = self._create_diskfile() - mkdirs(os.path.dirname(df._datadir)) - open(df._datadir, 'wb').close() - ohash = hash_path('a', 'c', 'o') - data_dir = ohash[-3:] - whole_path_from = os.path.join(self.objects, '0', data_dir) - orig_quarantine_renamer = diskfile.quarantine_renamer - called = [False] - - def wrapped(*args, **kwargs): - called[0] = True - return orig_quarantine_renamer(*args, **kwargs) - - try: - diskfile.quarantine_renamer = wrapped - diskfile.hash_suffix(whole_path_from, 101) - finally: - diskfile.quarantine_renamer = orig_quarantine_renamer - self.assertTrue(called[0]) - - def test_hash_suffix_one_file(self): - df = self._create_diskfile() - mkdirs(df._datadir) - f = open( - os.path.join(df._datadir, - Timestamp(time() - 100).internal + '.ts'), - 'wb') - f.write('1234567890') - f.close() - ohash = hash_path('a', 'c', 'o') - data_dir = ohash[-3:] - whole_path_from = os.path.join(self.objects, '0', data_dir) - diskfile.hash_suffix(whole_path_from, 101) - self.assertEquals(len(os.listdir(self.parts['0'])), 1) - - diskfile.hash_suffix(whole_path_from, 99) - self.assertEquals(len(os.listdir(self.parts['0'])), 0) - - def test_hash_suffix_oserror_on_hcl(self): - df = self._create_diskfile() - mkdirs(df._datadir) - f = open( - os.path.join(df._datadir, - Timestamp(time() - 100).internal + '.ts'), - 'wb') - f.write('1234567890') - f.close() - ohash = hash_path('a', 'c', 'o') - data_dir = ohash[-3:] - whole_path_from = os.path.join(self.objects, '0', data_dir) - state = [0] - orig_os_listdir = os.listdir - - def mock_os_listdir(*args, **kwargs): - # We want the first call to os.listdir() to succeed, which is the - # one directly from hash_suffix() itself, but then we want to fail - # the next call to os.listdir() which is from - # hash_cleanup_listdir() - if state[0] == 1: - raise OSError(errno.EACCES, os.strerror(errno.EACCES)) - state[0] = 1 - return orig_os_listdir(*args, **kwargs) - - with mock.patch('os.listdir', mock_os_listdir): - self.assertRaises(OSError, diskfile.hash_suffix, whole_path_from, - 101) - - def test_hash_suffix_multi_file_one(self): - df = self._create_diskfile() - mkdirs(df._datadir) - for tdiff in [1, 50, 100, 500]: - for suff in ['.meta', '.data', '.ts']: - f = open( - os.path.join( - df._datadir, - Timestamp(int(time()) - tdiff).internal + suff), - 'wb') - f.write('1234567890') - f.close() - - ohash = hash_path('a', 'c', 'o') - data_dir = ohash[-3:] - whole_path_from = os.path.join(self.objects, '0', data_dir) - hsh_path = os.listdir(whole_path_from)[0] - whole_hsh_path = os.path.join(whole_path_from, hsh_path) - - diskfile.hash_suffix(whole_path_from, 99) - # only the tombstone should be left - self.assertEquals(len(os.listdir(whole_hsh_path)), 1) - - def test_hash_suffix_multi_file_two(self): - df = self._create_diskfile() - mkdirs(df._datadir) - for tdiff in [1, 50, 100, 500]: - suffs = ['.meta', '.data'] - if tdiff > 50: - suffs.append('.ts') - for suff in suffs: - f = open( - os.path.join( - df._datadir, - Timestamp(int(time()) - tdiff).internal + suff), - 'wb') - f.write('1234567890') - f.close() - - ohash = hash_path('a', 'c', 'o') - data_dir = ohash[-3:] - whole_path_from = os.path.join(self.objects, '0', data_dir) - hsh_path = os.listdir(whole_path_from)[0] - whole_hsh_path = os.path.join(whole_path_from, hsh_path) - - diskfile.hash_suffix(whole_path_from, 99) - # only the meta and data should be left - self.assertEquals(len(os.listdir(whole_hsh_path)), 2) - - def test_hash_suffix_hsh_path_disappearance(self): - orig_rmdir = os.rmdir - - def _rmdir(path): - # Done twice to recreate what happens when it doesn't exist. - orig_rmdir(path) - orig_rmdir(path) - - df = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o') - mkdirs(df._datadir) - ohash = hash_path('a', 'c', 'o') - suffix = ohash[-3:] - suffix_path = os.path.join(self.objects, '0', suffix) - with mock.patch('os.rmdir', _rmdir): - # If hash_suffix doesn't handle the exception _rmdir will raise, - # this test will fail. - diskfile.hash_suffix(suffix_path, 123) - - def test_invalidate_hash(self): - - def assertFileData(file_path, data): - with open(file_path, 'r') as fp: - fdata = fp.read() - self.assertEquals(pickle.loads(fdata), pickle.loads(data)) - - df = self._create_diskfile() - mkdirs(df._datadir) - ohash = hash_path('a', 'c', 'o') - data_dir = ohash[-3:] - whole_path_from = os.path.join(self.objects, '0', data_dir) - hashes_file = os.path.join(self.objects, '0', - diskfile.HASH_FILE) - # test that non existent file except caught - self.assertEquals(diskfile.invalidate_hash(whole_path_from), - None) - # test that hashes get cleared - check_pickle_data = pickle.dumps({data_dir: None}, - diskfile.PICKLE_PROTOCOL) - for data_hash in [{data_dir: None}, {data_dir: 'abcdefg'}]: - with open(hashes_file, 'wb') as fp: - pickle.dump(data_hash, fp, diskfile.PICKLE_PROTOCOL) - diskfile.invalidate_hash(whole_path_from) - assertFileData(hashes_file, check_pickle_data) - - def test_invalidate_hash_bad_pickle(self): - df = self._create_diskfile() - mkdirs(df._datadir) - ohash = hash_path('a', 'c', 'o') - data_dir = ohash[-3:] - whole_path_from = os.path.join(self.objects, '0', data_dir) - hashes_file = os.path.join(self.objects, '0', - diskfile.HASH_FILE) - for data_hash in [{data_dir: None}, {data_dir: 'abcdefg'}]: - with open(hashes_file, 'wb') as fp: - fp.write('bad hash data') - try: - diskfile.invalidate_hash(whole_path_from) - except Exception as err: - self.fail("Unexpected exception raised: %s" % err) - else: - pass - - def test_get_hashes(self): - df = self._create_diskfile() - mkdirs(df._datadir) - with open( - os.path.join(df._datadir, - Timestamp(time()).internal + '.ts'), - 'wb') as f: - f.write('1234567890') - part = os.path.join(self.objects, '0') - hashed, hashes = diskfile.get_hashes(part) - self.assertEquals(hashed, 1) - self.assert_('a83' in hashes) - hashed, hashes = diskfile.get_hashes(part, do_listdir=True) - self.assertEquals(hashed, 0) - self.assert_('a83' in hashes) - hashed, hashes = diskfile.get_hashes(part, recalculate=['a83']) - self.assertEquals(hashed, 1) - self.assert_('a83' in hashes) - - def test_get_hashes_bad_dir(self): - df = self._create_diskfile() - mkdirs(df._datadir) - with open(os.path.join(self.objects, '0', 'bad'), 'wb') as f: - f.write('1234567890') - part = os.path.join(self.objects, '0') - hashed, hashes = diskfile.get_hashes(part) - self.assertEquals(hashed, 1) - self.assert_('a83' in hashes) - self.assert_('bad' not in hashes) - - def test_get_hashes_unmodified(self): - df = self._create_diskfile() - mkdirs(df._datadir) - with open( - os.path.join(df._datadir, - Timestamp(time()).internal + '.ts'), - 'wb') as f: - f.write('1234567890') - part = os.path.join(self.objects, '0') - hashed, hashes = diskfile.get_hashes(part) - i = [0] - - def _getmtime(filename): - i[0] += 1 - return 1 - with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): - hashed, hashes = diskfile.get_hashes( - part, recalculate=['a83']) - self.assertEquals(i[0], 2) - - def test_get_hashes_unmodified_norecalc(self): - df = self._create_diskfile() - mkdirs(df._datadir) - with open( - os.path.join(df._datadir, - Timestamp(time()).internal + '.ts'), - 'wb') as f: - f.write('1234567890') - part = os.path.join(self.objects, '0') - hashed, hashes_0 = diskfile.get_hashes(part) - self.assertEqual(hashed, 1) - self.assertTrue('a83' in hashes_0) - hashed, hashes_1 = diskfile.get_hashes(part) - self.assertEqual(hashed, 0) - self.assertTrue('a83' in hashes_0) - self.assertEqual(hashes_1, hashes_0) - - def test_get_hashes_hash_suffix_error(self): - df = self._create_diskfile() - mkdirs(df._datadir) - with open( - os.path.join(df._datadir, - Timestamp(time()).internal + '.ts'), - 'wb') as f: - f.write('1234567890') - part = os.path.join(self.objects, '0') - mocked_hash_suffix = mock.MagicMock( - side_effect=OSError(errno.EACCES, os.strerror(errno.EACCES))) - with mock.patch('swift.obj.diskfile.hash_suffix', mocked_hash_suffix): - hashed, hashes = diskfile.get_hashes(part) - self.assertEqual(hashed, 0) - self.assertEqual(hashes, {'a83': None}) - - def test_get_hashes_unmodified_and_zero_bytes(self): - df = self._create_diskfile() - mkdirs(df._datadir) - part = os.path.join(self.objects, '0') - open(os.path.join(part, diskfile.HASH_FILE), 'w') - # Now the hash file is zero bytes. - i = [0] - - def _getmtime(filename): - i[0] += 1 - return 1 - with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): - hashed, hashes = diskfile.get_hashes( - part, recalculate=[]) - # getmtime will actually not get called. Initially, the pickle.load - # will raise an exception first and later, force_rewrite will - # short-circuit the if clause to determine whether to write out a - # fresh hashes_file. - self.assertEquals(i[0], 0) - self.assertTrue('a83' in hashes) - - def test_get_hashes_modified(self): - df = self._create_diskfile() - mkdirs(df._datadir) - with open( - os.path.join(df._datadir, - Timestamp(time()).internal + '.ts'), - 'wb') as f: - f.write('1234567890') - part = os.path.join(self.objects, '0') - hashed, hashes = diskfile.get_hashes(part) - i = [0] - - def _getmtime(filename): - if i[0] < 3: - i[0] += 1 - return i[0] - with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): - hashed, hashes = diskfile.get_hashes( - part, recalculate=['a83']) - self.assertEquals(i[0], 3) - - def check_hash_cleanup_listdir(self, input_files, output_files): - orig_unlink = os.unlink - file_list = list(input_files) - - def mock_listdir(path): - return list(file_list) - - def mock_unlink(path): - # timestamp 1 is a special tag to pretend a file disappeared while - # working. - if '/0000000001.00000.' in path: - # Using actual os.unlink to reproduce exactly what OSError it - # raises. - orig_unlink(uuid.uuid4().hex) - file_list.remove(os.path.basename(path)) - - with unit_mock({'os.listdir': mock_listdir, 'os.unlink': mock_unlink}): - self.assertEquals(diskfile.hash_cleanup_listdir('/whatever'), - output_files) - - def test_hash_cleanup_listdir_purge_data_newer_ts(self): - # purge .data if there's a newer .ts - file1 = Timestamp(time()).internal + '.data' - file2 = Timestamp(time() + 1).internal + '.ts' - file_list = [file1, file2] - self.check_hash_cleanup_listdir(file_list, [file2]) - - def test_hash_cleanup_listdir_purge_ts_newer_data(self): - # purge .ts if there's a newer .data - file1 = Timestamp(time()).internal + '.ts' - file2 = Timestamp(time() + 1).internal + '.data' - file_list = [file1, file2] - self.check_hash_cleanup_listdir(file_list, [file2]) - - def test_hash_cleanup_listdir_keep_meta_data_purge_ts(self): - # keep .meta and .data if meta newer than data and purge .ts - file1 = Timestamp(time()).internal + '.ts' - file2 = Timestamp(time() + 1).internal + '.data' - file3 = Timestamp(time() + 2).internal + '.meta' - file_list = [file1, file2, file3] - self.check_hash_cleanup_listdir(file_list, [file3, file2]) - - def test_hash_cleanup_listdir_keep_one_ts(self): - # keep only latest of multiple .ts files - file1 = Timestamp(time()).internal + '.ts' - file2 = Timestamp(time() + 1).internal + '.ts' - file3 = Timestamp(time() + 2).internal + '.ts' - file_list = [file1, file2, file3] - self.check_hash_cleanup_listdir(file_list, [file3]) - - def test_hash_cleanup_listdir_keep_one_data(self): - # keep only latest of multiple .data files - file1 = Timestamp(time()).internal + '.data' - file2 = Timestamp(time() + 1).internal + '.data' - file3 = Timestamp(time() + 2).internal + '.data' - file_list = [file1, file2, file3] - self.check_hash_cleanup_listdir(file_list, [file3]) - - def test_hash_cleanup_listdir_keep_one_meta(self): - # keep only latest of multiple .meta files - file1 = Timestamp(time()).internal + '.data' - file2 = Timestamp(time() + 1).internal + '.meta' - file3 = Timestamp(time() + 2).internal + '.meta' - file_list = [file1, file2, file3] - self.check_hash_cleanup_listdir(file_list, [file3, file1]) - - def test_hash_cleanup_listdir_ignore_orphaned_ts(self): - # A more recent orphaned .meta file will prevent old .ts files - # from being cleaned up otherwise - file1 = Timestamp(time()).internal + '.ts' - file2 = Timestamp(time() + 1).internal + '.ts' - file3 = Timestamp(time() + 2).internal + '.meta' - file_list = [file1, file2, file3] - self.check_hash_cleanup_listdir(file_list, [file3, file2]) - - def test_hash_cleanup_listdir_purge_old_data_only(self): - # Oldest .data will be purge, .meta and .ts won't be touched - file1 = Timestamp(time()).internal + '.data' - file2 = Timestamp(time() + 1).internal + '.ts' - file3 = Timestamp(time() + 2).internal + '.meta' - file_list = [file1, file2, file3] - self.check_hash_cleanup_listdir(file_list, [file3, file2]) - - def test_hash_cleanup_listdir_purge_old_ts(self): - # A single old .ts file will be removed - file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.ts' - file_list = [file1] - self.check_hash_cleanup_listdir(file_list, []) - - def test_hash_cleanup_listdir_meta_keeps_old_ts(self): - # An orphaned .meta will not clean up a very old .ts - file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.ts' - file2 = Timestamp(time() + 2).internal + '.meta' - file_list = [file1, file2] - self.check_hash_cleanup_listdir(file_list, [file2, file1]) - - def test_hash_cleanup_listdir_keep_single_old_data(self): - # A single old .data file will not be removed - file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.data' - file_list = [file1] - self.check_hash_cleanup_listdir(file_list, [file1]) - - def test_hash_cleanup_listdir_keep_single_old_meta(self): - # A single old .meta file will not be removed - file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.meta' - file_list = [file1] - self.check_hash_cleanup_listdir(file_list, [file1]) - - def test_hash_cleanup_listdir_disappeared_path(self): - # Next line listing a non-existent dir used to propagate the OSError; - # now should mute that. - self.assertEqual(diskfile.hash_cleanup_listdir(uuid.uuid4().hex), []) - - def test_hash_cleanup_listdir_disappeared_before_unlink_1(self): - # Timestamp 1 makes other test routines pretend the file disappeared - # while working. - file1 = '0000000001.00000.ts' - file_list = [file1] - self.check_hash_cleanup_listdir(file_list, []) - - def test_hash_cleanup_listdir_disappeared_before_unlink_2(self): - # Timestamp 1 makes other test routines pretend the file disappeared - # while working. - file1 = '0000000001.00000.data' - file2 = '0000000002.00000.ts' - file_list = [file1, file2] - self.check_hash_cleanup_listdir(file_list, [file2]) - @patch_policies class TestObjectAuditLocationGenerator(unittest.TestCase): @@ -677,7 +266,8 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): pass def test_audit_location_class(self): - al = diskfile.AuditLocation('abc', '123', '_-_') + al = diskfile.AuditLocation('abc', '123', '_-_', + policy=POLICIES.legacy) self.assertEqual(str(al), 'abc') def test_finding_of_hashdirs(self): @@ -705,6 +295,7 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): "6c3", "fcd938702024c25fef6c32fef05298eb")) os.makedirs(os.path.join(tmpdir, "sdq", "objects-fud", "foo")) + os.makedirs(os.path.join(tmpdir, "sdq", "objects-+1", "foo")) self._make_file(os.path.join(tmpdir, "sdp", "objects", "1519", "fed")) @@ -723,7 +314,7 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): "4f9eee668b66c6f0250bfa3c7ab9e51e")) logger = debug_logger() - locations = [(loc.path, loc.device, loc.partition) + locations = [(loc.path, loc.device, loc.partition, loc.policy) for loc in diskfile.object_audit_location_generator( devices=tmpdir, mount_check=False, logger=logger)] @@ -732,44 +323,42 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): # expect some warnings about those bad dirs warnings = logger.get_lines_for_level('warning') self.assertEqual(set(warnings), set([ - 'Directory objects- does not map to a valid policy', - 'Directory objects-2 does not map to a valid policy', - 'Directory objects-99 does not map to a valid policy', - 'Directory objects-fud does not map to a valid policy'])) + ("Directory 'objects-' does not map to a valid policy " + "(Unknown policy, for index '')"), + ("Directory 'objects-2' does not map to a valid policy " + "(Unknown policy, for index '2')"), + ("Directory 'objects-99' does not map to a valid policy " + "(Unknown policy, for index '99')"), + ("Directory 'objects-fud' does not map to a valid policy " + "(Unknown policy, for index 'fud')"), + ("Directory 'objects-+1' does not map to a valid policy " + "(Unknown policy, for index '+1')"), + ])) expected = \ [(os.path.join(tmpdir, "sdp", "objects-1", "9970", "ca5", "4a943bc72c2e647c4675923d58cf4ca5"), - "sdp", "9970"), + "sdp", "9970", POLICIES[1]), (os.path.join(tmpdir, "sdp", "objects", "1519", "aca", "5c1fdc1ffb12e5eaf84edc30d8b67aca"), - "sdp", "1519"), + "sdp", "1519", POLICIES[0]), (os.path.join(tmpdir, "sdp", "objects", "1519", "aca", "fdfd184d39080020bc8b487f8a7beaca"), - "sdp", "1519"), + "sdp", "1519", POLICIES[0]), (os.path.join(tmpdir, "sdp", "objects", "1519", "df2", "b0fe7af831cc7b1af5bf486b1c841df2"), - "sdp", "1519"), + "sdp", "1519", POLICIES[0]), (os.path.join(tmpdir, "sdp", "objects", "9720", "ca5", "4a943bc72c2e647c4675923d58cf4ca5"), - "sdp", "9720"), - (os.path.join(tmpdir, "sdq", "objects-", "1135", "6c3", - "fcd938702024c25fef6c32fef05298eb"), - "sdq", "1135"), - (os.path.join(tmpdir, "sdq", "objects-2", "9971", "8eb", - "fcd938702024c25fef6c32fef05298eb"), - "sdq", "9971"), - (os.path.join(tmpdir, "sdq", "objects-99", "9972", "8eb", - "fcd938702024c25fef6c32fef05298eb"), - "sdq", "9972"), + "sdp", "9720", POLICIES[0]), (os.path.join(tmpdir, "sdq", "objects", "3071", "8eb", "fcd938702024c25fef6c32fef05298eb"), - "sdq", "3071"), + "sdq", "3071", POLICIES[0]), ] self.assertEqual(locations, expected) # now without a logger - locations = [(loc.path, loc.device, loc.partition) + locations = [(loc.path, loc.device, loc.partition, loc.policy) for loc in diskfile.object_audit_location_generator( devices=tmpdir, mount_check=False)] locations.sort() @@ -789,7 +378,7 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): "4993d582f41be9771505a8d4cb237a10")) locations = [ - (loc.path, loc.device, loc.partition) + (loc.path, loc.device, loc.partition, loc.policy) for loc in diskfile.object_audit_location_generator( devices=tmpdir, mount_check=True)] locations.sort() @@ -799,12 +388,12 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): [(os.path.join(tmpdir, "sdp", "objects", "2607", "df3", "ec2871fe724411f91787462f97d30df3"), - "sdp", "2607")]) + "sdp", "2607", POLICIES[0])]) # Do it again, this time with a logger. ml = mock.MagicMock() locations = [ - (loc.path, loc.device, loc.partition) + (loc.path, loc.device, loc.partition, loc.policy) for loc in diskfile.object_audit_location_generator( devices=tmpdir, mount_check=True, logger=ml)] ml.debug.assert_called_once_with( @@ -817,7 +406,7 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): # only normal FS corruption should be skipped over silently. def list_locations(dirname): - return [(loc.path, loc.device, loc.partition) + return [(loc.path, loc.device, loc.partition, loc.policy) for loc in diskfile.object_audit_location_generator( devices=dirname, mount_check=False)] @@ -843,7 +432,45 @@ class TestObjectAuditLocationGenerator(unittest.TestCase): self.assertRaises(OSError, list_locations, tmpdir) -class TestDiskFileManager(unittest.TestCase): +class TestDiskFileRouter(unittest.TestCase): + + def test_register(self): + with mock.patch.dict( + diskfile.DiskFileRouter.policy_type_to_manager_cls, {}): + @diskfile.DiskFileRouter.register('test-policy') + class TestDiskFileManager(diskfile.DiskFileManager): + pass + + @BaseStoragePolicy.register('test-policy') + class TestStoragePolicy(BaseStoragePolicy): + pass + + with patch_policies([TestStoragePolicy(0, 'test')]): + router = diskfile.DiskFileRouter({}, debug_logger('test')) + manager = router[POLICIES.default] + self.assertTrue(isinstance(manager, TestDiskFileManager)) + + +class BaseDiskFileTestMixin(object): + """ + Bag of helpers that are useful in the per-policy DiskFile test classes. + """ + + def _manager_mock(self, manager_attribute_name, df=None): + mgr_cls = df._manager.__class__ if df else self.mgr_cls + return '.'.join([ + mgr_cls.__module__, mgr_cls.__name__, manager_attribute_name]) + + +class DiskFileManagerMixin(BaseDiskFileTestMixin): + """ + Abstract test method mixin for concrete test cases - this class + won't get picked up by test runners because it doesn't subclass + unittest.TestCase and doesn't have [Tt]est in the name. + """ + + # set mgr_cls on subclasses + mgr_cls = None def setUp(self): self.tmpdir = mkdtemp() @@ -851,17 +478,111 @@ class TestDiskFileManager(unittest.TestCase): self.tmpdir, 'tmp_test_obj_server_DiskFile') self.existing_device1 = 'sda1' self.existing_device2 = 'sda2' - mkdirs(os.path.join(self.testdir, self.existing_device1, 'tmp')) - mkdirs(os.path.join(self.testdir, self.existing_device2, 'tmp')) + for policy in POLICIES: + mkdirs(os.path.join(self.testdir, self.existing_device1, + diskfile.get_tmp_dir(policy))) + mkdirs(os.path.join(self.testdir, self.existing_device2, + diskfile.get_tmp_dir(policy))) self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) self.conf = dict(devices=self.testdir, mount_check='false', keep_cache_size=2 * 1024) - self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) + self.logger = debug_logger('test-' + self.__class__.__name__) + self.df_mgr = self.mgr_cls(self.conf, self.logger) + self.df_router = diskfile.DiskFileRouter(self.conf, self.logger) def tearDown(self): rmtree(self.tmpdir, ignore_errors=1) + def _get_diskfile(self, policy, frag_index=None): + df_mgr = self.df_router[policy] + return df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o', + policy=policy, frag_index=frag_index) + + def _test_get_ondisk_files(self, scenarios, policy, + frag_index=None): + class_under_test = self._get_diskfile(policy, frag_index=frag_index) + with mock.patch('swift.obj.diskfile.os.listdir', + lambda _: []): + self.assertEqual((None, None, None), + class_under_test._get_ondisk_file()) + + returned_ext_order = ('.data', '.meta', '.ts') + for test in scenarios: + chosen = dict((f[1], os.path.join(class_under_test._datadir, f[0])) + for f in test if f[1]) + expected = tuple(chosen.get(ext) for ext in returned_ext_order) + files = list(zip(*test)[0]) + for _order in ('ordered', 'shuffled', 'shuffled'): + class_under_test = self._get_diskfile(policy, frag_index) + try: + with mock.patch('swift.obj.diskfile.os.listdir', + lambda _: files): + actual = class_under_test._get_ondisk_file() + self.assertEqual(expected, actual, + 'Expected %s from %s but got %s' + % (expected, files, actual)) + except AssertionError as e: + self.fail('%s with files %s' % (str(e), files)) + shuffle(files) + + def _test_hash_cleanup_listdir_files(self, scenarios, policy, + reclaim_age=None): + # check that expected files are left in hashdir after cleanup + for test in scenarios: + class_under_test = self.df_router[policy] + files = list(zip(*test)[0]) + hashdir = os.path.join(self.testdir, str(uuid.uuid4())) + os.mkdir(hashdir) + for fname in files: + open(os.path.join(hashdir, fname), 'w') + expected_after_cleanup = set([f[0] for f in test + if (f[2] if len(f) > 2 else f[1])]) + if reclaim_age: + class_under_test.hash_cleanup_listdir( + hashdir, reclaim_age=reclaim_age) + else: + with mock.patch('swift.obj.diskfile.time') as mock_time: + # don't reclaim anything + mock_time.time.return_value = 0.0 + class_under_test.hash_cleanup_listdir(hashdir) + after_cleanup = set(os.listdir(hashdir)) + errmsg = "expected %r, got %r for test %r" % ( + sorted(expected_after_cleanup), sorted(after_cleanup), test + ) + self.assertEqual(expected_after_cleanup, after_cleanup, errmsg) + + def _test_yield_hashes_cleanup(self, scenarios, policy): + # opportunistic test to check that yield_hashes cleans up dir using + # same scenarios as passed to _test_hash_cleanup_listdir_files + for test in scenarios: + class_under_test = self.df_router[policy] + files = list(zip(*test)[0]) + dev_path = os.path.join(self.testdir, str(uuid.uuid4())) + hashdir = os.path.join( + dev_path, diskfile.get_data_dir(policy), + '0', 'abc', '9373a92d072897b136b3fc06595b4abc') + os.makedirs(hashdir) + for fname in files: + open(os.path.join(hashdir, fname), 'w') + expected_after_cleanup = set([f[0] for f in test + if f[1] or len(f) > 2 and f[2]]) + with mock.patch('swift.obj.diskfile.time') as mock_time: + # don't reclaim anything + mock_time.time.return_value = 0.0 + mock_func = 'swift.obj.diskfile.DiskFileManager.get_dev_path' + with mock.patch(mock_func) as mock_path: + mock_path.return_value = dev_path + for _ in class_under_test.yield_hashes( + 'ignored', '0', policy, suffixes=['abc']): + # return values are tested in test_yield_hashes_* + pass + after_cleanup = set(os.listdir(hashdir)) + errmsg = "expected %r, got %r for test %r" % ( + sorted(expected_after_cleanup), sorted(after_cleanup), test + ) + self.assertEqual(expected_after_cleanup, after_cleanup, errmsg) + def test_construct_dev_path(self): res_path = self.df_mgr.construct_dev_path('abc') self.assertEqual(os.path.join(self.df_mgr.devices, 'abc'), res_path) @@ -872,12 +593,13 @@ class TestDiskFileManager(unittest.TestCase): with mock.patch('swift.obj.diskfile.write_pickle') as wp: self.df_mgr.pickle_async_update(self.existing_device1, 'a', 'c', 'o', - dict(a=1, b=2), ts, 0) + dict(a=1, b=2), ts, POLICIES[0]) dp = self.df_mgr.construct_dev_path(self.existing_device1) ohash = diskfile.hash_path('a', 'c', 'o') wp.assert_called_with({'a': 1, 'b': 2}, - os.path.join(dp, diskfile.get_async_dir(0), - ohash[-3:], ohash + '-' + ts), + os.path.join( + dp, diskfile.get_async_dir(POLICIES[0]), + ohash[-3:], ohash + '-' + ts), os.path.join(dp, 'tmp')) self.df_mgr.logger.increment.assert_called_with('async_pendings') @@ -885,32 +607,16 @@ class TestDiskFileManager(unittest.TestCase): locations = list(self.df_mgr.object_audit_location_generator()) self.assertEqual(locations, []) - def test_get_hashes_bad_dev(self): - self.df_mgr.mount_check = True - with mock.patch('swift.obj.diskfile.check_mount', - mock.MagicMock(side_effect=[False])): - self.assertRaises(DiskFileDeviceUnavailable, - self.df_mgr.get_hashes, 'sdb1', '0', '123', - 'objects') - - def test_get_hashes_w_nothing(self): - hashes = self.df_mgr.get_hashes(self.existing_device1, '0', '123', '0') - self.assertEqual(hashes, {}) - # get_hashes creates the partition path, so call again for code - # path coverage, ensuring the result is unchanged - hashes = self.df_mgr.get_hashes(self.existing_device1, '0', '123', '0') - self.assertEqual(hashes, {}) - def test_replication_lock_on(self): # Double check settings self.df_mgr.replication_one_per_device = True self.df_mgr.replication_lock_timeout = 0.1 dev_path = os.path.join(self.testdir, self.existing_device1) - with self.df_mgr.replication_lock(dev_path): + with self.df_mgr.replication_lock(self.existing_device1): lock_exc = None exc = None try: - with self.df_mgr.replication_lock(dev_path): + with self.df_mgr.replication_lock(self.existing_device1): raise Exception( '%r was not replication locked!' % dev_path) except ReplicationLockTimeout as err: @@ -943,12 +649,10 @@ class TestDiskFileManager(unittest.TestCase): # Double check settings self.df_mgr.replication_one_per_device = True self.df_mgr.replication_lock_timeout = 0.1 - dev_path = os.path.join(self.testdir, self.existing_device1) - dev_path2 = os.path.join(self.testdir, self.existing_device2) - with self.df_mgr.replication_lock(dev_path): + with self.df_mgr.replication_lock(self.existing_device1): lock_exc = None try: - with self.df_mgr.replication_lock(dev_path2): + with self.df_mgr.replication_lock(self.existing_device2): pass except ReplicationLockTimeout as err: lock_exc = err @@ -965,10 +669,1094 @@ class TestDiskFileManager(unittest.TestCase): self.assertTrue('splice()' in warnings[-1]) self.assertFalse(mgr.use_splice) + def test_get_diskfile_from_hash_dev_path_fail(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + hclistdir.return_value = ['1381679759.90941.data'] + readmeta.return_value = {'name': '/a/c/o'} + self.assertRaises( + DiskFileDeviceUnavailable, + self.df_mgr.get_diskfile_from_hash, + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + + def test_get_diskfile_from_hash_not_dir(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata'), + mock.patch(self._manager_mock('quarantine_renamer'))) as \ + (dfclass, hclistdir, readmeta, quarantine_renamer): + osexc = OSError() + osexc.errno = errno.ENOTDIR + hclistdir.side_effect = osexc + readmeta.return_value = {'name': '/a/c/o'} + self.assertRaises( + DiskFileNotExist, + self.df_mgr.get_diskfile_from_hash, + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + quarantine_renamer.assert_called_once_with( + '/srv/dev/', + '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900') + + def test_get_diskfile_from_hash_no_dir(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + osexc = OSError() + osexc.errno = errno.ENOENT + hclistdir.side_effect = osexc + readmeta.return_value = {'name': '/a/c/o'} + self.assertRaises( + DiskFileNotExist, + self.df_mgr.get_diskfile_from_hash, + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + + def test_get_diskfile_from_hash_other_oserror(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + osexc = OSError() + hclistdir.side_effect = osexc + readmeta.return_value = {'name': '/a/c/o'} + self.assertRaises( + OSError, + self.df_mgr.get_diskfile_from_hash, + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + + def test_get_diskfile_from_hash_no_actual_files(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + hclistdir.return_value = [] + readmeta.return_value = {'name': '/a/c/o'} + self.assertRaises( + DiskFileNotExist, + self.df_mgr.get_diskfile_from_hash, + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + + def test_get_diskfile_from_hash_read_metadata_problem(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + hclistdir.return_value = ['1381679759.90941.data'] + readmeta.side_effect = EOFError() + self.assertRaises( + DiskFileNotExist, + self.df_mgr.get_diskfile_from_hash, + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + + def test_get_diskfile_from_hash_no_meta_name(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + hclistdir.return_value = ['1381679759.90941.data'] + readmeta.return_value = {} + try: + self.df_mgr.get_diskfile_from_hash( + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', + POLICIES[0]) + except DiskFileNotExist as err: + exc = err + self.assertEqual(str(exc), '') + + def test_get_diskfile_from_hash_bad_meta_name(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + hclistdir.return_value = ['1381679759.90941.data'] + readmeta.return_value = {'name': 'bad'} + try: + self.df_mgr.get_diskfile_from_hash( + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', + POLICIES[0]) + except DiskFileNotExist as err: + exc = err + self.assertEqual(str(exc), '') + + def test_get_diskfile_from_hash(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with nested( + mock.patch(self._manager_mock('diskfile_cls')), + mock.patch(self._manager_mock('hash_cleanup_listdir')), + mock.patch('swift.obj.diskfile.read_metadata')) as \ + (dfclass, hclistdir, readmeta): + hclistdir.return_value = ['1381679759.90941.data'] + readmeta.return_value = {'name': '/a/c/o'} + self.df_mgr.get_diskfile_from_hash( + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + dfclass.assert_called_once_with( + self.df_mgr, '/srv/dev/', self.df_mgr.threadpools['dev'], '9', + 'a', 'c', 'o', policy=POLICIES[0]) + hclistdir.assert_called_once_with( + '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900', + 604800) + readmeta.assert_called_once_with( + '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900/' + '1381679759.90941.data') + + def test_listdir_enoent(self): + oserror = OSError() + oserror.errno = errno.ENOENT + self.df_mgr.logger.error = mock.MagicMock() + with mock.patch('os.listdir', side_effect=oserror): + self.assertEqual(self.df_mgr._listdir('path'), []) + self.assertEqual(self.df_mgr.logger.error.mock_calls, []) + + def test_listdir_other_oserror(self): + oserror = OSError() + self.df_mgr.logger.error = mock.MagicMock() + with mock.patch('os.listdir', side_effect=oserror): + self.assertEqual(self.df_mgr._listdir('path'), []) + self.df_mgr.logger.error.assert_called_once_with( + 'ERROR: Skipping %r due to error with listdir attempt: %s', + 'path', oserror) + + def test_listdir(self): + self.df_mgr.logger.error = mock.MagicMock() + with mock.patch('os.listdir', return_value=['abc', 'def']): + self.assertEqual(self.df_mgr._listdir('path'), ['abc', 'def']) + self.assertEqual(self.df_mgr.logger.error.mock_calls, []) + + def test_yield_suffixes_dev_path_fail(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) + exc = None + try: + list(self.df_mgr.yield_suffixes(self.existing_device1, '9', 0)) + except DiskFileDeviceUnavailable as err: + exc = err + self.assertEqual(str(exc), '') + + def test_yield_suffixes(self): + self.df_mgr._listdir = mock.MagicMock(return_value=[ + 'abc', 'def', 'ghi', 'abcd', '012']) + dev = self.existing_device1 + self.assertEqual( + list(self.df_mgr.yield_suffixes(dev, '9', POLICIES[0])), + [(self.testdir + '/' + dev + '/objects/9/abc', 'abc'), + (self.testdir + '/' + dev + '/objects/9/def', 'def'), + (self.testdir + '/' + dev + '/objects/9/012', '012')]) + + def test_yield_hashes_dev_path_fail(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) + exc = None + try: + list(self.df_mgr.yield_hashes(self.existing_device1, '9', + POLICIES[0])) + except DiskFileDeviceUnavailable as err: + exc = err + self.assertEqual(str(exc), '') + + def test_yield_hashes_empty(self): + def _listdir(path): + return [] + + with mock.patch('os.listdir', _listdir): + self.assertEqual(list(self.df_mgr.yield_hashes( + self.existing_device1, '9', POLICIES[0])), []) + + def test_yield_hashes_empty_suffixes(self): + def _listdir(path): + return [] + + with mock.patch('os.listdir', _listdir): + self.assertEqual( + list(self.df_mgr.yield_hashes(self.existing_device1, '9', + POLICIES[0], + suffixes=['456'])), []) + + def _check_yield_hashes(self, policy, suffix_map, expected, **kwargs): + device = self.existing_device1 + part = '9' + part_path = os.path.join( + self.testdir, device, diskfile.get_data_dir(policy), part) + + def _listdir(path): + if path == part_path: + return suffix_map.keys() + for suff, hash_map in suffix_map.items(): + if path == os.path.join(part_path, suff): + return hash_map.keys() + for hash_, files in hash_map.items(): + if path == os.path.join(part_path, suff, hash_): + return files + self.fail('Unexpected listdir of %r' % path) + expected_items = [ + (os.path.join(part_path, hash_[-3:], hash_), hash_, + Timestamp(ts).internal) + for hash_, ts in expected.items()] + with nested( + mock.patch('os.listdir', _listdir), + mock.patch('os.unlink')): + df_mgr = self.df_router[policy] + hash_items = list(df_mgr.yield_hashes( + device, part, policy, **kwargs)) + expected = sorted(expected_items) + actual = sorted(hash_items) + self.assertEqual(actual, expected, + 'Expected %s but got %s' % (expected, actual)) + + def test_yield_hashes_tombstones(self): + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + ts2 = next(ts_iter) + ts3 = next(ts_iter) + suffix_map = { + '27e': { + '1111111111111111111111111111127e': [ + ts1.internal + '.ts'], + '2222222222222222222222222222227e': [ + ts2.internal + '.ts'], + }, + 'd41': { + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaad41': [] + }, + 'd98': {}, + '00b': { + '3333333333333333333333333333300b': [ + ts1.internal + '.ts', + ts2.internal + '.ts', + ts3.internal + '.ts', + ] + }, + '204': { + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbb204': [ + ts3.internal + '.ts', + ] + } + } + expected = { + '1111111111111111111111111111127e': ts1.internal, + '2222222222222222222222222222227e': ts2.internal, + '3333333333333333333333333333300b': ts3.internal, + } + for policy in POLICIES: + self._check_yield_hashes(policy, suffix_map, expected, + suffixes=['27e', '00b']) + @patch_policies -class TestDiskFile(unittest.TestCase): - """Test swift.obj.diskfile.DiskFile""" +class TestDiskFileManager(DiskFileManagerMixin, unittest.TestCase): + + mgr_cls = diskfile.DiskFileManager + + def test_get_ondisk_files_with_repl_policy(self): + # Each scenario specifies a list of (filename, extension) tuples. If + # extension is set then that filename should be returned by the method + # under test for that extension type. + scenarios = [[('0000000007.00000.data', '.data')], + + [('0000000007.00000.ts', '.ts')], + + # older tombstone is ignored + [('0000000007.00000.ts', '.ts'), + ('0000000006.00000.ts', False)], + + # older data is ignored + [('0000000007.00000.data', '.data'), + ('0000000006.00000.data', False), + ('0000000004.00000.ts', False)], + + # newest meta trumps older meta + [('0000000009.00000.meta', '.meta'), + ('0000000008.00000.meta', False), + ('0000000007.00000.data', '.data'), + ('0000000004.00000.ts', False)], + + # meta older than data is ignored + [('0000000007.00000.data', '.data'), + ('0000000006.00000.meta', False), + ('0000000004.00000.ts', False)], + + # meta without data is ignored + [('0000000007.00000.meta', False, True), + ('0000000006.00000.ts', '.ts'), + ('0000000004.00000.data', False)], + + # tombstone trumps meta and data at same timestamp + [('0000000006.00000.meta', False), + ('0000000006.00000.ts', '.ts'), + ('0000000006.00000.data', False)], + ] + + self._test_get_ondisk_files(scenarios, POLICIES[0], None) + self._test_hash_cleanup_listdir_files(scenarios, POLICIES[0]) + self._test_yield_hashes_cleanup(scenarios, POLICIES[0]) + + def test_get_ondisk_files_with_stray_meta(self): + # get_ondisk_files does not tolerate a stray .meta file + + class_under_test = self._get_diskfile(POLICIES[0]) + files = ['0000000007.00000.meta'] + + self.assertRaises(AssertionError, + class_under_test.manager.get_ondisk_files, files, + self.testdir) + + def test_yield_hashes(self): + old_ts = '1383180000.12345' + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal + suffix_map = { + 'abc': { + '9373a92d072897b136b3fc06595b4abc': [ + fresh_ts + '.ts'], + }, + '456': { + '9373a92d072897b136b3fc06595b0456': [ + old_ts + '.data'], + '9373a92d072897b136b3fc06595b7456': [ + fresh_ts + '.ts', + fresher_ts + '.data'], + }, + 'def': {}, + } + expected = { + '9373a92d072897b136b3fc06595b4abc': fresh_ts, + '9373a92d072897b136b3fc06595b0456': old_ts, + '9373a92d072897b136b3fc06595b7456': fresher_ts, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected) + + def test_yield_hashes_yields_meta_timestamp(self): + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + ts2 = next(ts_iter) + ts3 = next(ts_iter) + suffix_map = { + 'abc': { + '9373a92d072897b136b3fc06595b4abc': [ + ts1.internal + '.ts', + ts2.internal + '.meta'], + }, + '456': { + '9373a92d072897b136b3fc06595b0456': [ + ts1.internal + '.data', + ts2.internal + '.meta', + ts3.internal + '.meta'], + '9373a92d072897b136b3fc06595b7456': [ + ts1.internal + '.data', + ts2.internal + '.meta'], + }, + } + expected = { + '9373a92d072897b136b3fc06595b4abc': ts2, + '9373a92d072897b136b3fc06595b0456': ts3, + '9373a92d072897b136b3fc06595b7456': ts2, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected) + + def test_yield_hashes_suffix_filter(self): + # test again with limited suffixes + old_ts = '1383180000.12345' + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal + suffix_map = { + 'abc': { + '9373a92d072897b136b3fc06595b4abc': [ + fresh_ts + '.ts'], + }, + '456': { + '9373a92d072897b136b3fc06595b0456': [ + old_ts + '.data'], + '9373a92d072897b136b3fc06595b7456': [ + fresh_ts + '.ts', + fresher_ts + '.data'], + }, + 'def': {}, + } + expected = { + '9373a92d072897b136b3fc06595b0456': old_ts, + '9373a92d072897b136b3fc06595b7456': fresher_ts, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + suffixes=['456']) + + def test_yield_hashes_fails_with_bad_ondisk_filesets(self): + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + suffix_map = { + '456': { + '9373a92d072897b136b3fc06595b0456': [ + ts1.internal + '.data'], + '9373a92d072897b136b3fc06595ba456': [ + ts1.internal + '.meta'], + }, + } + expected = { + '9373a92d072897b136b3fc06595b0456': ts1, + } + try: + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + self.fail('Expected AssertionError') + except AssertionError: + pass + + +@patch_policies(with_ec_default=True) +class TestECDiskFileManager(DiskFileManagerMixin, unittest.TestCase): + + mgr_cls = diskfile.ECDiskFileManager + + def test_get_ondisk_files_with_ec_policy(self): + # Each scenario specifies a list of (filename, extension, [survives]) + # tuples. If extension is set then that filename should be returned by + # the method under test for that extension type. If the optional + # 'survives' is True, the filename should still be in the dir after + # cleanup. + scenarios = [[('0000000007.00000.ts', '.ts')], + + [('0000000007.00000.ts', '.ts'), + ('0000000006.00000.ts', False)], + + # highest frag index is chosen by default + [('0000000007.00000.durable', '.durable'), + ('0000000007.00000#1.data', '.data'), + ('0000000007.00000#0.data', False, True)], + + # data with no durable is ignored + [('0000000007.00000#0.data', False, True)], + + # data newer than durable is ignored + [('0000000008.00000#1.data', False, True), + ('0000000007.00000.durable', '.durable'), + ('0000000007.00000#1.data', '.data'), + ('0000000007.00000#0.data', False, True)], + + # data newer than durable ignored, even if its only data + [('0000000008.00000#1.data', False, True), + ('0000000007.00000.durable', False, False)], + + # data older than durable is ignored + [('0000000007.00000.durable', '.durable'), + ('0000000007.00000#1.data', '.data'), + ('0000000006.00000#1.data', False), + ('0000000004.00000.ts', False)], + + # data older than durable ignored, even if its only data + [('0000000007.00000.durable', False, False), + ('0000000006.00000#1.data', False), + ('0000000004.00000.ts', False)], + + # newer meta trumps older meta + [('0000000009.00000.meta', '.meta'), + ('0000000008.00000.meta', False), + ('0000000007.00000.durable', '.durable'), + ('0000000007.00000#14.data', '.data'), + ('0000000004.00000.ts', False)], + + # older meta is ignored + [('0000000007.00000.durable', '.durable'), + ('0000000007.00000#14.data', '.data'), + ('0000000006.00000.meta', False), + ('0000000004.00000.ts', False)], + + # tombstone trumps meta, data, durable at older timestamp + [('0000000006.00000.ts', '.ts'), + ('0000000005.00000.meta', False), + ('0000000004.00000.durable', False), + ('0000000004.00000#0.data', False)], + + # tombstone trumps meta, data, durable at same timestamp + [('0000000006.00000.meta', False), + ('0000000006.00000.ts', '.ts'), + ('0000000006.00000.durable', False), + ('0000000006.00000#0.data', False)], + + # missing durable invalidates data + [('0000000006.00000.meta', False, True), + ('0000000006.00000#0.data', False, True)] + ] + + self._test_get_ondisk_files(scenarios, POLICIES.default, None) + self._test_hash_cleanup_listdir_files(scenarios, POLICIES.default) + self._test_yield_hashes_cleanup(scenarios, POLICIES.default) + + def test_get_ondisk_files_with_ec_policy_and_frag_index(self): + # Each scenario specifies a list of (filename, extension) tuples. If + # extension is set then that filename should be returned by the method + # under test for that extension type. + scenarios = [[('0000000007.00000#2.data', False, True), + ('0000000007.00000#1.data', '.data'), + ('0000000007.00000#0.data', False, True), + ('0000000007.00000.durable', '.durable')], + + # specific frag newer than durable is ignored + [('0000000007.00000#2.data', False, True), + ('0000000007.00000#1.data', False, True), + ('0000000007.00000#0.data', False, True), + ('0000000006.00000.durable', '.durable')], + + # specific frag older than durable is ignored + [('0000000007.00000#2.data', False), + ('0000000007.00000#1.data', False), + ('0000000007.00000#0.data', False), + ('0000000008.00000.durable', '.durable')], + + # specific frag older than newest durable is ignored + # even if is also has a durable + [('0000000007.00000#2.data', False), + ('0000000007.00000#1.data', False), + ('0000000007.00000.durable', False), + ('0000000008.00000#0.data', False), + ('0000000008.00000.durable', '.durable')], + + # meta included when frag index is specified + [('0000000009.00000.meta', '.meta'), + ('0000000007.00000#2.data', False, True), + ('0000000007.00000#1.data', '.data'), + ('0000000007.00000#0.data', False, True), + ('0000000007.00000.durable', '.durable')], + + # specific frag older than tombstone is ignored + [('0000000009.00000.ts', '.ts'), + ('0000000007.00000#2.data', False), + ('0000000007.00000#1.data', False), + ('0000000007.00000#0.data', False), + ('0000000007.00000.durable', False)], + + # no data file returned if specific frag index missing + [('0000000007.00000#2.data', False, True), + ('0000000007.00000#14.data', False, True), + ('0000000007.00000#0.data', False, True), + ('0000000007.00000.durable', '.durable')], + + # meta ignored if specific frag index missing + [('0000000008.00000.meta', False, True), + ('0000000007.00000#14.data', False, True), + ('0000000007.00000#0.data', False, True), + ('0000000007.00000.durable', '.durable')], + + # meta ignored if no data files + # Note: this is anomalous, because we are specifying a + # frag_index, get_ondisk_files will tolerate .meta with + # no .data + [('0000000088.00000.meta', False, True), + ('0000000077.00000.durable', '.durable')] + ] + + self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index=1) + # note: not calling self._test_hash_cleanup_listdir_files(scenarios, 0) + # here due to the anomalous scenario as commented above + + def test_hash_cleanup_listdir_reclaim(self): + # Each scenario specifies a list of (filename, extension, [survives]) + # tuples. If extension is set or 'survives' is True, the filename + # should still be in the dir after cleanup. + much_older = Timestamp(time() - 2000).internal + older = Timestamp(time() - 1001).internal + newer = Timestamp(time() - 900).internal + scenarios = [[('%s.ts' % older, False, False)], + + # fresh tombstone is preserved + [('%s.ts' % newer, '.ts', True)], + + # isolated .durable is cleaned up immediately + [('%s.durable' % newer, False, False)], + + # ...even when other older files are in dir + [('%s.durable' % older, False, False), + ('%s.ts' % much_older, False, False)], + + # isolated .data files are cleaned up when stale + [('%s#2.data' % older, False, False), + ('%s#4.data' % older, False, False)], + + # ...even when there is an older durable fileset + [('%s#2.data' % older, False, False), + ('%s#4.data' % older, False, False), + ('%s#2.data' % much_older, '.data', True), + ('%s#4.data' % much_older, False, True), + ('%s.durable' % much_older, '.durable', True)], + + # ... but preserved if still fresh + [('%s#2.data' % newer, False, True), + ('%s#4.data' % newer, False, True)], + + # ... and we could have a mixture of fresh and stale .data + [('%s#2.data' % newer, False, True), + ('%s#4.data' % older, False, False)], + + # TODO these remaining scenarios exhibit different + # behavior than the legacy replication DiskFileManager + # behavior... + + # tombstone reclaimed despite newer non-durable data + [('%s#2.data' % newer, False, True), + ('%s#4.data' % older, False, False), + ('%s.ts' % much_older, '.ts', False)], + + # tombstone reclaimed despite newer non-durable data + [('%s.ts' % older, '.ts', False), + ('%s.durable' % much_older, False, False)], + + # tombstone reclaimed despite junk file + [('junk', False, True), + ('%s.ts' % much_older, '.ts', False)], + ] + + self._test_hash_cleanup_listdir_files(scenarios, POLICIES.default, + reclaim_age=1000) + + def test_get_ondisk_files_with_stray_meta(self): + # get_ondisk_files does not tolerate a stray .meta file + scenarios = [['0000000007.00000.meta'], + + ['0000000007.00000.meta', + '0000000006.00000.durable'], + + ['0000000007.00000.meta', + '0000000006.00000#1.data'], + + ['0000000007.00000.meta', + '0000000006.00000.durable', + '0000000005.00000#1.data'] + ] + for files in scenarios: + class_under_test = self._get_diskfile(POLICIES.default) + self.assertRaises(DiskFileNotExist, class_under_test.open) + + def test_parse_on_disk_filename(self): + mgr = self.df_router[POLICIES.default] + for ts in (Timestamp('1234567890.00001'), + Timestamp('1234567890.00001', offset=17)): + for frag in (0, 2, 14): + fname = '%s#%s.data' % (ts.internal, frag) + info = mgr.parse_on_disk_filename(fname) + self.assertEqual(ts, info['timestamp']) + self.assertEqual(frag, info['frag_index']) + self.assertEqual(mgr.make_on_disk_filename(**info), fname) + + for ext in ('.meta', '.durable', '.ts'): + fname = '%s%s' % (ts.internal, ext) + info = mgr.parse_on_disk_filename(fname) + self.assertEqual(ts, info['timestamp']) + self.assertEqual(None, info['frag_index']) + self.assertEqual(mgr.make_on_disk_filename(**info), fname) + + def test_parse_on_disk_filename_errors(self): + mgr = self.df_router[POLICIES.default] + for ts in (Timestamp('1234567890.00001'), + Timestamp('1234567890.00001', offset=17)): + fname = '%s.data' % ts.internal + try: + mgr.parse_on_disk_filename(fname) + msg = 'Expected DiskFileError for filename %s' % fname + self.fail(msg) + except DiskFileError: + pass + + expected = { + '': 'bad', + 'foo': 'bad', + '1.314': 'bad', + 1.314: 'bad', + -2: 'negative', + '-2': 'negative', + None: 'bad', + 'None': 'bad', + } + + for frag, msg in expected.items(): + fname = '%s#%s.data' % (ts.internal, frag) + try: + mgr.parse_on_disk_filename(fname) + except DiskFileError as e: + self.assertTrue(msg in str(e).lower()) + else: + msg = 'Expected DiskFileError for filename %s' % fname + self.fail(msg) + + def test_make_on_disk_filename(self): + mgr = self.df_router[POLICIES.default] + for ts in (Timestamp('1234567890.00001'), + Timestamp('1234567890.00001', offset=17)): + for frag in (0, '0', 2, '2', 14, '14'): + expected = '%s#%s.data' % (ts.internal, frag) + actual = mgr.make_on_disk_filename( + ts, '.data', frag_index=frag) + self.assertEqual(expected, actual) + parsed = mgr.parse_on_disk_filename(actual) + self.assertEqual(parsed, { + 'timestamp': ts, + 'frag_index': int(frag), + 'ext': '.data', + }) + # these functions are inverse + self.assertEqual( + mgr.make_on_disk_filename(**parsed), + expected) + + for ext in ('.meta', '.durable', '.ts'): + expected = '%s%s' % (ts.internal, ext) + # frag index should not be required + actual = mgr.make_on_disk_filename(ts, ext) + self.assertEqual(expected, actual) + # frag index should be ignored + actual = mgr.make_on_disk_filename( + ts, ext, frag_index=frag) + self.assertEqual(expected, actual) + parsed = mgr.parse_on_disk_filename(actual) + self.assertEqual(parsed, { + 'timestamp': ts, + 'frag_index': None, + 'ext': ext, + }) + # these functions are inverse + self.assertEqual( + mgr.make_on_disk_filename(**parsed), + expected) + + actual = mgr.make_on_disk_filename(ts) + self.assertEqual(ts, actual) + + def test_make_on_disk_filename_with_bad_frag_index(self): + mgr = self.df_router[POLICIES.default] + ts = Timestamp('1234567890.00001') + try: + # .data requires a frag_index kwarg + mgr.make_on_disk_filename(ts, '.data') + self.fail('Expected DiskFileError for missing frag_index') + except DiskFileError: + pass + + for frag in (None, 'foo', '1.314', 1.314, -2, '-2'): + try: + mgr.make_on_disk_filename(ts, '.data', frag_index=frag) + self.fail('Expected DiskFileError for frag_index %s' % frag) + except DiskFileError: + pass + for ext in ('.meta', '.durable', '.ts'): + expected = '%s%s' % (ts.internal, ext) + # bad frag index should be ignored + actual = mgr.make_on_disk_filename(ts, ext, frag_index=frag) + self.assertEqual(expected, actual) + + def test_is_obsolete(self): + mgr = self.df_router[POLICIES.default] + for ts in (Timestamp('1234567890.00001'), + Timestamp('1234567890.00001', offset=17)): + for ts2 in (Timestamp('1234567890.99999'), + Timestamp('1234567890.99999', offset=17), + ts): + f_2 = mgr.make_on_disk_filename(ts, '.durable') + for fi in (0, 2): + for ext in ('.data', '.meta', '.durable', '.ts'): + f_1 = mgr.make_on_disk_filename( + ts2, ext, frag_index=fi) + self.assertFalse(mgr.is_obsolete(f_1, f_2), + '%s should not be obsolete w.r.t. %s' + % (f_1, f_2)) + + for ts2 in (Timestamp('1234567890.00000'), + Timestamp('1234500000.00000', offset=0), + Timestamp('1234500000.00000', offset=17)): + f_2 = mgr.make_on_disk_filename(ts, '.durable') + for fi in (0, 2): + for ext in ('.data', '.meta', '.durable', '.ts'): + f_1 = mgr.make_on_disk_filename( + ts2, ext, frag_index=fi) + self.assertTrue(mgr.is_obsolete(f_1, f_2), + '%s should not be w.r.t. %s' + % (f_1, f_2)) + + def test_yield_hashes(self): + old_ts = '1383180000.12345' + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal + suffix_map = { + 'abc': { + '9373a92d072897b136b3fc06595b4abc': [ + fresh_ts + '.ts'], + }, + '456': { + '9373a92d072897b136b3fc06595b0456': [ + old_ts + '#2.data', + old_ts + '.durable'], + '9373a92d072897b136b3fc06595b7456': [ + fresh_ts + '.ts', + fresher_ts + '#2.data', + fresher_ts + '.durable'], + }, + 'def': {}, + } + expected = { + '9373a92d072897b136b3fc06595b4abc': fresh_ts, + '9373a92d072897b136b3fc06595b0456': old_ts, + '9373a92d072897b136b3fc06595b7456': fresher_ts, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + + def test_yield_hashes_yields_meta_timestamp(self): + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + ts2 = next(ts_iter) + ts3 = next(ts_iter) + suffix_map = { + 'abc': { + '9373a92d072897b136b3fc06595b4abc': [ + ts1.internal + '.ts', + ts2.internal + '.meta'], + }, + '456': { + '9373a92d072897b136b3fc06595b0456': [ + ts1.internal + '#2.data', + ts1.internal + '.durable', + ts2.internal + '.meta', + ts3.internal + '.meta'], + '9373a92d072897b136b3fc06595b7456': [ + ts1.internal + '#2.data', + ts1.internal + '.durable', + ts2.internal + '.meta'], + }, + } + expected = { + # TODO: differs from repl DiskFileManager which *will* + # return meta timestamp when only meta and ts on disk + '9373a92d072897b136b3fc06595b4abc': ts1, + '9373a92d072897b136b3fc06595b0456': ts3, + '9373a92d072897b136b3fc06595b7456': ts2, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected) + + # but meta timestamp is not returned if specified frag index + # is not found + expected = { + # TODO: differs from repl DiskFileManager which *will* + # return meta timestamp when only meta and ts on disk + '9373a92d072897b136b3fc06595b4abc': ts1, + '9373a92d072897b136b3fc06595b0456': ts3, + '9373a92d072897b136b3fc06595b7456': ts2, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=3) + + def test_yield_hashes_suffix_filter(self): + # test again with limited suffixes + old_ts = '1383180000.12345' + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal + suffix_map = { + 'abc': { + '9373a92d072897b136b3fc06595b4abc': [ + fresh_ts + '.ts'], + }, + '456': { + '9373a92d072897b136b3fc06595b0456': [ + old_ts + '#2.data', + old_ts + '.durable'], + '9373a92d072897b136b3fc06595b7456': [ + fresh_ts + '.ts', + fresher_ts + '#2.data', + fresher_ts + '.durable'], + }, + 'def': {}, + } + expected = { + '9373a92d072897b136b3fc06595b0456': old_ts, + '9373a92d072897b136b3fc06595b7456': fresher_ts, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + suffixes=['456'], frag_index=2) + + def test_yield_hashes_skips_missing_durable(self): + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + suffix_map = { + '456': { + '9373a92d072897b136b3fc06595b0456': [ + ts1.internal + '#2.data', + ts1.internal + '.durable'], + '9373a92d072897b136b3fc06595b7456': [ + ts1.internal + '#2.data'], + }, + } + expected = { + '9373a92d072897b136b3fc06595b0456': ts1, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + + # if we add a durable it shows up + suffix_map['456']['9373a92d072897b136b3fc06595b7456'].append( + ts1.internal + '.durable') + expected = { + '9373a92d072897b136b3fc06595b0456': ts1, + '9373a92d072897b136b3fc06595b7456': ts1, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + + def test_yield_hashes_skips_data_without_durable(self): + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + ts2 = next(ts_iter) + ts3 = next(ts_iter) + suffix_map = { + '456': { + '9373a92d072897b136b3fc06595b0456': [ + ts1.internal + '#2.data', + ts1.internal + '.durable', + ts2.internal + '#2.data', + ts3.internal + '#2.data'], + }, + } + expected = { + '9373a92d072897b136b3fc06595b0456': ts1, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=None) + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + + # if we add a durable then newer data shows up + suffix_map['456']['9373a92d072897b136b3fc06595b0456'].append( + ts2.internal + '.durable') + expected = { + '9373a92d072897b136b3fc06595b0456': ts2, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=None) + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + + def test_yield_hashes_ignores_bad_ondisk_filesets(self): + # this differs from DiskFileManager.yield_hashes which will fail + # when encountering a bad on-disk file set + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + ts2 = next(ts_iter) + suffix_map = { + '456': { + '9373a92d072897b136b3fc06595b0456': [ + ts1.internal + '#2.data', + ts1.internal + '.durable'], + '9373a92d072897b136b3fc06595b7456': [ + ts1.internal + '.data'], + '9373a92d072897b136b3fc06595b8456': [ + 'junk_file'], + '9373a92d072897b136b3fc06595b9456': [ + ts1.internal + '.data', + ts2.internal + '.meta'], + '9373a92d072897b136b3fc06595ba456': [ + ts1.internal + '.meta'], + '9373a92d072897b136b3fc06595bb456': [ + ts1.internal + '.meta', + ts2.internal + '.meta'], + }, + } + expected = { + '9373a92d072897b136b3fc06595b0456': ts1, + '9373a92d072897b136b3fc06595ba456': ts1, + '9373a92d072897b136b3fc06595bb456': ts2, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + + def test_yield_hashes_filters_frag_index(self): + ts_iter = (Timestamp(t) for t in itertools.count(int(time()))) + ts1 = next(ts_iter) + ts2 = next(ts_iter) + ts3 = next(ts_iter) + suffix_map = { + '27e': { + '1111111111111111111111111111127e': [ + ts1.internal + '#2.data', + ts1.internal + '#3.data', + ts1.internal + '.durable', + ], + '2222222222222222222222222222227e': [ + ts1.internal + '#2.data', + ts1.internal + '.durable', + ts2.internal + '#2.data', + ts2.internal + '.durable', + ], + }, + 'd41': { + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaad41': [ + ts1.internal + '#3.data', + ts1.internal + '.durable', + ], + }, + '00b': { + '3333333333333333333333333333300b': [ + ts1.internal + '#2.data', + ts2.internal + '#2.data', + ts3.internal + '#2.data', + ts3.internal + '.durable', + ], + }, + } + expected = { + '1111111111111111111111111111127e': ts1, + '2222222222222222222222222222227e': ts2, + '3333333333333333333333333333300b': ts3, + } + self._check_yield_hashes(POLICIES.default, suffix_map, expected, + frag_index=2) + + def test_get_diskfile_from_hash_frag_index_filter(self): + df = self._get_diskfile(POLICIES.default) + hash_ = os.path.basename(df._datadir) + self.assertRaises(DiskFileNotExist, + self.df_mgr.get_diskfile_from_hash, + self.existing_device1, '0', hash_, + POLICIES.default) # sanity + frag_index = 7 + timestamp = Timestamp(time()) + for frag_index in (4, 7): + with df.create() as writer: + data = 'test_data' + writer.write(data) + metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': timestamp.internal, + 'Content-Length': len(data), + 'X-Object-Sysmeta-Ec-Frag-Index': str(frag_index), + } + writer.put(metadata) + writer.commit(timestamp) + + df4 = self.df_mgr.get_diskfile_from_hash( + self.existing_device1, '0', hash_, POLICIES.default, frag_index=4) + self.assertEqual(df4._frag_index, 4) + self.assertEqual( + df4.read_metadata()['X-Object-Sysmeta-Ec-Frag-Index'], '4') + df7 = self.df_mgr.get_diskfile_from_hash( + self.existing_device1, '0', hash_, POLICIES.default, frag_index=7) + self.assertEqual(df7._frag_index, 7) + self.assertEqual( + df7.read_metadata()['X-Object-Sysmeta-Ec-Frag-Index'], '7') + + +class DiskFileMixin(BaseDiskFileTestMixin): + + # set mgr_cls on subclasses + mgr_cls = None def setUp(self): """Set up for testing swift.obj.diskfile""" @@ -978,12 +1766,22 @@ class TestDiskFile(unittest.TestCase): self.existing_device = 'sda1' for policy in POLICIES: mkdirs(os.path.join(self.testdir, self.existing_device, - get_tmp_dir(policy.idx))) + diskfile.get_tmp_dir(policy))) self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) self.conf = dict(devices=self.testdir, mount_check='false', keep_cache_size=2 * 1024, mb_per_sync=1) - self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) + self.logger = debug_logger('test-' + self.__class__.__name__) + self.df_mgr = self.mgr_cls(self.conf, self.logger) + self.df_router = diskfile.DiskFileRouter(self.conf, self.logger) + self._ts_iter = (Timestamp(t) for t in + itertools.count(int(time()))) + + def ts(self): + """ + Timestamps - forever. + """ + return next(self._ts_iter) def tearDown(self): """Tear down for testing swift.obj.diskfile""" @@ -995,11 +1793,11 @@ class TestDiskFile(unittest.TestCase): mkdirs(df._datadir) if timestamp is None: timestamp = time() - timestamp = Timestamp(timestamp).internal + timestamp = Timestamp(timestamp) if not metadata: metadata = {} if 'X-Timestamp' not in metadata: - metadata['X-Timestamp'] = Timestamp(timestamp).internal + metadata['X-Timestamp'] = timestamp.internal if 'ETag' not in metadata: etag = md5() etag.update(data) @@ -1008,17 +1806,24 @@ class TestDiskFile(unittest.TestCase): metadata['name'] = '/a/c/o' if 'Content-Length' not in metadata: metadata['Content-Length'] = str(len(data)) - data_file = os.path.join(df._datadir, timestamp + ext) + filename = timestamp.internal + ext + if ext == '.data' and df.policy.policy_type == EC_POLICY: + filename = '%s#%s.data' % (timestamp.internal, df._frag_index) + data_file = os.path.join(df._datadir, filename) with open(data_file, 'wb') as f: f.write(data) xattr.setxattr(f.fileno(), diskfile.METADATA_KEY, pickle.dumps(metadata, diskfile.PICKLE_PROTOCOL)) def _simple_get_diskfile(self, partition='0', account='a', container='c', - obj='o', policy_idx=0): - return self.df_mgr.get_diskfile(self.existing_device, - partition, account, container, obj, - policy_idx) + obj='o', policy=None, frag_index=None): + policy = policy or POLICIES.default + df_mgr = self.df_router[policy] + if policy.policy_type == EC_POLICY and frag_index is None: + frag_index = 2 + return df_mgr.get_diskfile(self.existing_device, partition, + account, container, obj, + policy=policy, frag_index=frag_index) def _create_test_file(self, data, timestamp=None, metadata=None, account='a', container='c', obj='o'): @@ -1027,12 +1832,62 @@ class TestDiskFile(unittest.TestCase): metadata.setdefault('name', '/%s/%s/%s' % (account, container, obj)) df = self._simple_get_diskfile(account=account, container=container, obj=obj) - self._create_ondisk_file(df, data, timestamp, metadata) - df = self._simple_get_diskfile(account=account, container=container, - obj=obj) + if timestamp is None: + timestamp = time() + timestamp = Timestamp(timestamp) + with df.create() as writer: + new_metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': timestamp.internal, + 'Content-Length': len(data), + } + new_metadata.update(metadata) + writer.write(data) + writer.put(new_metadata) + writer.commit(timestamp) df.open() return df + def test_get_dev_path(self): + self.df_mgr.devices = '/srv' + device = 'sda1' + dev_path = os.path.join(self.df_mgr.devices, device) + + mount_check = None + self.df_mgr.mount_check = True + with mock.patch('swift.obj.diskfile.check_mount', + mock.MagicMock(return_value=False)): + self.assertEqual(self.df_mgr.get_dev_path(device, mount_check), + None) + with mock.patch('swift.obj.diskfile.check_mount', + mock.MagicMock(return_value=True)): + self.assertEqual(self.df_mgr.get_dev_path(device, mount_check), + dev_path) + + self.df_mgr.mount_check = False + with mock.patch('swift.obj.diskfile.check_dir', + mock.MagicMock(return_value=False)): + self.assertEqual(self.df_mgr.get_dev_path(device, mount_check), + None) + with mock.patch('swift.obj.diskfile.check_dir', + mock.MagicMock(return_value=True)): + self.assertEqual(self.df_mgr.get_dev_path(device, mount_check), + dev_path) + + mount_check = True + with mock.patch('swift.obj.diskfile.check_mount', + mock.MagicMock(return_value=False)): + self.assertEqual(self.df_mgr.get_dev_path(device, mount_check), + None) + with mock.patch('swift.obj.diskfile.check_mount', + mock.MagicMock(return_value=True)): + self.assertEqual(self.df_mgr.get_dev_path(device, mount_check), + dev_path) + + mount_check = False + self.assertEqual(self.df_mgr.get_dev_path(device, mount_check), + dev_path) + def test_open_not_exist(self): df = self._simple_get_diskfile() self.assertRaises(DiskFileNotExist, df.open) @@ -1050,15 +1905,17 @@ class TestDiskFile(unittest.TestCase): self.fail("Unexpected swift exception raised: %r" % err) def test_get_metadata(self): - df = self._create_test_file('1234567890', timestamp=42) + timestamp = self.ts().internal + df = self._create_test_file('1234567890', timestamp=timestamp) md = df.get_metadata() - self.assertEqual(md['X-Timestamp'], Timestamp(42).internal) + self.assertEqual(md['X-Timestamp'], timestamp) def test_read_metadata(self): - self._create_test_file('1234567890', timestamp=42) + timestamp = self.ts().internal + self._create_test_file('1234567890', timestamp=timestamp) df = self._simple_get_diskfile() md = df.read_metadata() - self.assertEqual(md['X-Timestamp'], Timestamp(42).internal) + self.assertEqual(md['X-Timestamp'], timestamp) def test_read_metadata_no_xattr(self): def mock_getxattr(*args, **kargs): @@ -1086,15 +1943,16 @@ class TestDiskFile(unittest.TestCase): self.fail("Expected DiskFileNotOpen exception") def test_disk_file_default_disallowed_metadata(self): - # build an object with some meta (ts 41) + # build an object with some meta (at t0+1s) orig_metadata = {'X-Object-Meta-Key1': 'Value1', 'Content-Type': 'text/garbage'} - df = self._get_open_disk_file(ts=41, extra_metadata=orig_metadata) + df = self._get_open_disk_file(ts=self.ts().internal, + extra_metadata=orig_metadata) with df.open(): self.assertEquals('1024', df._metadata['Content-Length']) - # write some new metadata (fast POST, don't send orig meta, ts 42) + # write some new metadata (fast POST, don't send orig meta, at t0+1) df = self._simple_get_diskfile() - df.write_metadata({'X-Timestamp': Timestamp(42).internal, + df.write_metadata({'X-Timestamp': self.ts().internal, 'X-Object-Meta-Key2': 'Value2'}) df = self._simple_get_diskfile() with df.open(): @@ -1106,15 +1964,16 @@ class TestDiskFile(unittest.TestCase): self.assertEquals('Value2', df._metadata['X-Object-Meta-Key2']) def test_disk_file_preserves_sysmeta(self): - # build an object with some meta (ts 41) + # build an object with some meta (at t0) orig_metadata = {'X-Object-Sysmeta-Key1': 'Value1', 'Content-Type': 'text/garbage'} - df = self._get_open_disk_file(ts=41, extra_metadata=orig_metadata) + df = self._get_open_disk_file(ts=self.ts().internal, + extra_metadata=orig_metadata) with df.open(): self.assertEquals('1024', df._metadata['Content-Length']) - # write some new metadata (fast POST, don't send orig meta, ts 42) + # write some new metadata (fast POST, don't send orig meta, at t0+1s) df = self._simple_get_diskfile() - df.write_metadata({'X-Timestamp': Timestamp(42).internal, + df.write_metadata({'X-Timestamp': self.ts().internal, 'X-Object-Sysmeta-Key1': 'Value2', 'X-Object-Meta-Key3': 'Value3'}) df = self._simple_get_diskfile() @@ -1268,34 +2127,38 @@ class TestDiskFile(unittest.TestCase): def test_disk_file_mkstemp_creates_dir(self): for policy in POLICIES: tmpdir = os.path.join(self.testdir, self.existing_device, - get_tmp_dir(policy.idx)) + diskfile.get_tmp_dir(policy)) os.rmdir(tmpdir) - df = self._simple_get_diskfile(policy_idx=policy.idx) + df = self._simple_get_diskfile(policy=policy) with df.create(): self.assert_(os.path.exists(tmpdir)) def _get_open_disk_file(self, invalid_type=None, obj_name='o', fsize=1024, csize=8, mark_deleted=False, prealloc=False, - ts=None, mount_check=False, extra_metadata=None): + ts=None, mount_check=False, extra_metadata=None, + policy=None, frag_index=None): '''returns a DiskFile''' - df = self._simple_get_diskfile(obj=obj_name) + policy = policy or POLICIES.legacy + df = self._simple_get_diskfile(obj=obj_name, policy=policy, + frag_index=frag_index) data = '0' * fsize etag = md5() if ts: - timestamp = ts + timestamp = Timestamp(ts) else: - timestamp = Timestamp(time()).internal + timestamp = Timestamp(time()) if prealloc: prealloc_size = fsize else: prealloc_size = None + with df.create(size=prealloc_size) as writer: upload_size = writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, - 'X-Timestamp': timestamp, + 'X-Timestamp': timestamp.internal, 'Content-Length': str(upload_size), } metadata.update(extra_metadata or {}) @@ -1318,6 +2181,7 @@ class TestDiskFile(unittest.TestCase): elif invalid_type == 'Bad-X-Delete-At': metadata['X-Delete-At'] = 'bad integer' diskfile.write_metadata(writer._fd, metadata) + writer.commit(timestamp) if mark_deleted: df.delete(timestamp) @@ -1348,9 +2212,16 @@ class TestDiskFile(unittest.TestCase): self.conf['disk_chunk_size'] = csize self.conf['mount_check'] = mount_check - self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) - df = self._simple_get_diskfile(obj=obj_name) + self.df_mgr = self.mgr_cls(self.conf, self.logger) + self.df_router = diskfile.DiskFileRouter(self.conf, self.logger) + + # actual on disk frag_index may have been set by metadata + frag_index = metadata.get('X-Object-Sysmeta-Ec-Frag-Index', + frag_index) + df = self._simple_get_diskfile(obj=obj_name, policy=policy, + frag_index=frag_index) df.open() + if invalid_type == 'Zero-Byte': fp = open(df._data_file, 'w') fp.close() @@ -1576,7 +2447,7 @@ class TestDiskFile(unittest.TestCase): pass df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', - 'xyz') + 'xyz', policy=POLICIES.legacy) self.assertRaises(DiskFileQuarantined, df.open) # make sure the right thing got quarantined; the suffix dir should not @@ -1586,7 +2457,7 @@ class TestDiskFile(unittest.TestCase): def test_create_prealloc(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', - 'xyz') + 'xyz', policy=POLICIES.legacy) with mock.patch("swift.obj.diskfile.fallocate") as fa: with df.create(size=200) as writer: used_fd = writer._fd @@ -1594,7 +2465,7 @@ class TestDiskFile(unittest.TestCase): def test_create_prealloc_oserror(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', - 'xyz') + 'xyz', policy=POLICIES.legacy) for e in (errno.ENOSPC, errno.EDQUOT): with mock.patch("swift.obj.diskfile.fallocate", mock.MagicMock(side_effect=OSError( @@ -1621,7 +2492,7 @@ class TestDiskFile(unittest.TestCase): def test_create_mkstemp_no_space(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', - 'xyz') + 'xyz', policy=POLICIES.legacy) for e in (errno.ENOSPC, errno.EDQUOT): with mock.patch("swift.obj.diskfile.mkstemp", mock.MagicMock(side_effect=OSError( @@ -1648,7 +2519,7 @@ class TestDiskFile(unittest.TestCase): def test_create_close_oserror(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', - 'xyz') + 'xyz', policy=POLICIES.legacy) with mock.patch("swift.obj.diskfile.os.close", mock.MagicMock(side_effect=OSError( errno.EACCES, os.strerror(errno.EACCES)))): @@ -1662,11 +2533,12 @@ class TestDiskFile(unittest.TestCase): def test_write_metadata(self): df = self._create_test_file('1234567890') + file_count = len(os.listdir(df._datadir)) timestamp = Timestamp(time()).internal metadata = {'X-Timestamp': timestamp, 'X-Object-Meta-test': 'data'} df.write_metadata(metadata) dl = os.listdir(df._datadir) - self.assertEquals(len(dl), 2) + self.assertEquals(len(dl), file_count + 1) exp_name = '%s.meta' % timestamp self.assertTrue(exp_name in set(dl)) @@ -1704,14 +2576,135 @@ class TestDiskFile(unittest.TestCase): DiskFileNoSpace, diskfile.write_metadata, 'n/a', metadata) + def _create_diskfile_dir(self, timestamp, policy): + timestamp = Timestamp(timestamp) + df = self._simple_get_diskfile(account='a', container='c', + obj='o_%s' % policy, + policy=policy) + + with df.create() as writer: + metadata = { + 'ETag': 'bogus_etag', + 'X-Timestamp': timestamp.internal, + 'Content-Length': '0', + } + if policy.policy_type == EC_POLICY: + metadata['X-Object-Sysmeta-Ec-Frag-Index'] = \ + df._frag_index or 7 + writer.put(metadata) + writer.commit(timestamp) + return writer._datadir + + def test_commit(self): + for policy in POLICIES: + # create first fileset as starting state + timestamp = Timestamp(time()).internal + datadir = self._create_diskfile_dir(timestamp, policy) + dl = os.listdir(datadir) + expected = ['%s.data' % timestamp] + if policy.policy_type == EC_POLICY: + expected = ['%s#2.data' % timestamp, + '%s.durable' % timestamp] + self.assertEquals(len(dl), len(expected), + 'Unexpected dir listing %s' % dl) + self.assertEqual(sorted(expected), sorted(dl)) + + def test_write_cleanup(self): + for policy in POLICIES: + # create first fileset as starting state + timestamp_1 = Timestamp(time()).internal + datadir_1 = self._create_diskfile_dir(timestamp_1, policy) + # second write should clean up first fileset + timestamp_2 = Timestamp(time() + 1).internal + datadir_2 = self._create_diskfile_dir(timestamp_2, policy) + # sanity check + self.assertEqual(datadir_1, datadir_2) + dl = os.listdir(datadir_2) + expected = ['%s.data' % timestamp_2] + if policy.policy_type == EC_POLICY: + expected = ['%s#2.data' % timestamp_2, + '%s.durable' % timestamp_2] + self.assertEquals(len(dl), len(expected), + 'Unexpected dir listing %s' % dl) + self.assertEqual(sorted(expected), sorted(dl)) + + def test_commit_fsync(self): + for policy in POLICIES: + mock_fsync = mock.MagicMock() + df = self._simple_get_diskfile(account='a', container='c', + obj='o', policy=policy) + + timestamp = Timestamp(time()) + with df.create() as writer: + metadata = { + 'ETag': 'bogus_etag', + 'X-Timestamp': timestamp.internal, + 'Content-Length': '0', + } + writer.put(metadata) + with mock.patch('swift.obj.diskfile.fsync', mock_fsync): + writer.commit(timestamp) + expected = { + EC_POLICY: 1, + REPL_POLICY: 0, + }[policy.policy_type] + self.assertEqual(expected, mock_fsync.call_count) + if policy.policy_type == EC_POLICY: + durable_file = '%s.durable' % timestamp.internal + self.assertTrue(durable_file in str(mock_fsync.call_args[0])) + + def test_commit_ignores_hash_cleanup_listdir_error(self): + for policy in POLICIES: + # Check OSError from hash_cleanup_listdir is caught and ignored + mock_hcl = mock.MagicMock(side_effect=OSError) + df = self._simple_get_diskfile(account='a', container='c', + obj='o_hcl_error', policy=policy) + + timestamp = Timestamp(time()) + with df.create() as writer: + metadata = { + 'ETag': 'bogus_etag', + 'X-Timestamp': timestamp.internal, + 'Content-Length': '0', + } + writer.put(metadata) + with mock.patch(self._manager_mock( + 'hash_cleanup_listdir', df), mock_hcl): + writer.commit(timestamp) + expected = { + EC_POLICY: 1, + REPL_POLICY: 0, + }[policy.policy_type] + self.assertEqual(expected, mock_hcl.call_count) + expected = ['%s.data' % timestamp.internal] + if policy.policy_type == EC_POLICY: + expected = ['%s#2.data' % timestamp.internal, + '%s.durable' % timestamp.internal] + dl = os.listdir(df._datadir) + self.assertEquals(len(dl), len(expected), + 'Unexpected dir listing %s' % dl) + self.assertEqual(sorted(expected), sorted(dl)) + def test_delete(self): - df = self._get_open_disk_file() - ts = time() - df.delete(ts) - exp_name = '%s.ts' % Timestamp(ts).internal - dl = os.listdir(df._datadir) - self.assertEquals(len(dl), 1) - self.assertTrue(exp_name in set(dl)) + for policy in POLICIES: + if policy.policy_type == EC_POLICY: + metadata = {'X-Object-Sysmeta-Ec-Frag-Index': '1'} + fi = 1 + else: + metadata = {} + fi = None + df = self._get_open_disk_file(policy=policy, frag_index=fi, + extra_metadata=metadata) + + ts = Timestamp(time()) + df.delete(ts) + exp_name = '%s.ts' % ts.internal + dl = os.listdir(df._datadir) + self.assertEquals(len(dl), 1) + self.assertTrue(exp_name in set(dl), + 'Expected file %s missing in %s' % (exp_name, dl)) + # cleanup before next policy + os.unlink(os.path.join(df._datadir, exp_name)) def test_open_deleted(self): df = self._get_open_disk_file() @@ -1748,7 +2741,8 @@ class TestDiskFile(unittest.TestCase): 'blah blah', account='three', container='blind', obj='mice')._datadir df = self.df_mgr.get_diskfile_from_audit_location( - diskfile.AuditLocation(hashdir, self.existing_device, '0')) + diskfile.AuditLocation(hashdir, self.existing_device, '0', + policy=POLICIES.default)) df.open() self.assertEqual(df._name, '/three/blind/mice') @@ -1756,14 +2750,16 @@ class TestDiskFile(unittest.TestCase): hashdir = self._create_test_file( 'blah blah', account='this', container='is', obj='right')._datadir - - datafile = os.path.join(hashdir, os.listdir(hashdir)[0]) + datafilename = [f for f in os.listdir(hashdir) + if f.endswith('.data')][0] + datafile = os.path.join(hashdir, datafilename) meta = diskfile.read_metadata(datafile) meta['name'] = '/this/is/wrong' diskfile.write_metadata(datafile, meta) df = self.df_mgr.get_diskfile_from_audit_location( - diskfile.AuditLocation(hashdir, self.existing_device, '0')) + diskfile.AuditLocation(hashdir, self.existing_device, '0', + policy=POLICIES.default)) self.assertRaises(DiskFileQuarantined, df.open) def test_close_error(self): @@ -1778,7 +2774,10 @@ class TestDiskFile(unittest.TestCase): pass # close is called at the end of the iterator self.assertEquals(reader._fp, None) - self.assertEquals(len(df._logger.log_dict['error']), 1) + error_lines = df._logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + self.assertTrue('close failure' in error_lines[0]) + self.assertTrue('Bad' in error_lines[0]) def test_mount_checking(self): @@ -1829,6 +2828,9 @@ class TestDiskFile(unittest.TestCase): self._create_ondisk_file(df, '', ext='.meta', timestamp=9) self._create_ondisk_file(df, 'B', ext='.data', timestamp=8) self._create_ondisk_file(df, 'A', ext='.data', timestamp=7) + if df.policy.policy_type == EC_POLICY: + self._create_ondisk_file(df, '', ext='.durable', timestamp=8) + self._create_ondisk_file(df, '', ext='.durable', timestamp=7) self._create_ondisk_file(df, '', ext='.ts', timestamp=6) self._create_ondisk_file(df, '', ext='.ts', timestamp=5) df = self._simple_get_diskfile() @@ -1842,6 +2844,9 @@ class TestDiskFile(unittest.TestCase): df = self._simple_get_diskfile() self._create_ondisk_file(df, 'B', ext='.data', timestamp=10) self._create_ondisk_file(df, 'A', ext='.data', timestamp=9) + if df.policy.policy_type == EC_POLICY: + self._create_ondisk_file(df, '', ext='.durable', timestamp=10) + self._create_ondisk_file(df, '', ext='.durable', timestamp=9) self._create_ondisk_file(df, '', ext='.ts', timestamp=8) self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, '', ext='.meta', timestamp=6) @@ -1858,6 +2863,9 @@ class TestDiskFile(unittest.TestCase): self._create_ondisk_file(df, 'X', ext='.bar', timestamp=11) self._create_ondisk_file(df, 'B', ext='.data', timestamp=10) self._create_ondisk_file(df, 'A', ext='.data', timestamp=9) + if df.policy.policy_type == EC_POLICY: + self._create_ondisk_file(df, '', ext='.durable', timestamp=10) + self._create_ondisk_file(df, '', ext='.durable', timestamp=9) self._create_ondisk_file(df, '', ext='.ts', timestamp=8) self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, '', ext='.meta', timestamp=6) @@ -1879,6 +2887,9 @@ class TestDiskFile(unittest.TestCase): self._create_ondisk_file(df, 'X', ext='.bar', timestamp=11) self._create_ondisk_file(df, 'B', ext='.data', timestamp=10) self._create_ondisk_file(df, 'A', ext='.data', timestamp=9) + if df.policy.policy_type == EC_POLICY: + self._create_ondisk_file(df, '', ext='.durable', timestamp=10) + self._create_ondisk_file(df, '', ext='.durable', timestamp=9) self._create_ondisk_file(df, '', ext='.ts', timestamp=8) self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, '', ext='.meta', timestamp=6) @@ -1900,300 +2911,6 @@ class TestDiskFile(unittest.TestCase): log_lines = df._logger.get_lines_for_level('error') self.assert_('a very special error' in log_lines[-1]) - def test_get_diskfile_from_hash_dev_path_fail(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - hclistdir.return_value = ['1381679759.90941.data'] - readmeta.return_value = {'name': '/a/c/o'} - self.assertRaises( - DiskFileDeviceUnavailable, - self.df_mgr.get_diskfile_from_hash, - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - - def test_get_diskfile_from_hash_not_dir(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata'), - mock.patch('swift.obj.diskfile.quarantine_renamer')) as \ - (dfclass, hclistdir, readmeta, quarantine_renamer): - osexc = OSError() - osexc.errno = errno.ENOTDIR - hclistdir.side_effect = osexc - readmeta.return_value = {'name': '/a/c/o'} - self.assertRaises( - DiskFileNotExist, - self.df_mgr.get_diskfile_from_hash, - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - quarantine_renamer.assert_called_once_with( - '/srv/dev/', - '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900') - - def test_get_diskfile_from_hash_no_dir(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - osexc = OSError() - osexc.errno = errno.ENOENT - hclistdir.side_effect = osexc - readmeta.return_value = {'name': '/a/c/o'} - self.assertRaises( - DiskFileNotExist, - self.df_mgr.get_diskfile_from_hash, - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - - def test_get_diskfile_from_hash_other_oserror(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - osexc = OSError() - hclistdir.side_effect = osexc - readmeta.return_value = {'name': '/a/c/o'} - self.assertRaises( - OSError, - self.df_mgr.get_diskfile_from_hash, - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - - def test_get_diskfile_from_hash_no_actual_files(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - hclistdir.return_value = [] - readmeta.return_value = {'name': '/a/c/o'} - self.assertRaises( - DiskFileNotExist, - self.df_mgr.get_diskfile_from_hash, - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - - def test_get_diskfile_from_hash_read_metadata_problem(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - hclistdir.return_value = ['1381679759.90941.data'] - readmeta.side_effect = EOFError() - self.assertRaises( - DiskFileNotExist, - self.df_mgr.get_diskfile_from_hash, - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - - def test_get_diskfile_from_hash_no_meta_name(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - hclistdir.return_value = ['1381679759.90941.data'] - readmeta.return_value = {} - try: - self.df_mgr.get_diskfile_from_hash( - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - except DiskFileNotExist as err: - exc = err - self.assertEqual(str(exc), '') - - def test_get_diskfile_from_hash_bad_meta_name(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - hclistdir.return_value = ['1381679759.90941.data'] - readmeta.return_value = {'name': 'bad'} - try: - self.df_mgr.get_diskfile_from_hash( - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - except DiskFileNotExist as err: - exc = err - self.assertEqual(str(exc), '') - - def test_get_diskfile_from_hash(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') - with nested( - mock.patch('swift.obj.diskfile.DiskFile'), - mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), - mock.patch('swift.obj.diskfile.read_metadata')) as \ - (dfclass, hclistdir, readmeta): - hclistdir.return_value = ['1381679759.90941.data'] - readmeta.return_value = {'name': '/a/c/o'} - self.df_mgr.get_diskfile_from_hash( - 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', 0) - dfclass.assert_called_once_with( - self.df_mgr, '/srv/dev/', self.df_mgr.threadpools['dev'], '9', - 'a', 'c', 'o', policy_idx=0) - hclistdir.assert_called_once_with( - '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900', - 604800) - readmeta.assert_called_once_with( - '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900/' - '1381679759.90941.data') - - def test_listdir_enoent(self): - oserror = OSError() - oserror.errno = errno.ENOENT - self.df_mgr.logger.error = mock.MagicMock() - with mock.patch('os.listdir', side_effect=oserror): - self.assertEqual(self.df_mgr._listdir('path'), []) - self.assertEqual(self.df_mgr.logger.error.mock_calls, []) - - def test_listdir_other_oserror(self): - oserror = OSError() - self.df_mgr.logger.error = mock.MagicMock() - with mock.patch('os.listdir', side_effect=oserror): - self.assertEqual(self.df_mgr._listdir('path'), []) - self.df_mgr.logger.error.assert_called_once_with( - 'ERROR: Skipping %r due to error with listdir attempt: %s', - 'path', oserror) - - def test_listdir(self): - self.df_mgr.logger.error = mock.MagicMock() - with mock.patch('os.listdir', return_value=['abc', 'def']): - self.assertEqual(self.df_mgr._listdir('path'), ['abc', 'def']) - self.assertEqual(self.df_mgr.logger.error.mock_calls, []) - - def test_yield_suffixes_dev_path_fail(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) - exc = None - try: - list(self.df_mgr.yield_suffixes('dev', '9', 0)) - except DiskFileDeviceUnavailable as err: - exc = err - self.assertEqual(str(exc), '') - - def test_yield_suffixes(self): - self.df_mgr._listdir = mock.MagicMock(return_value=[ - 'abc', 'def', 'ghi', 'abcd', '012']) - self.assertEqual( - list(self.df_mgr.yield_suffixes('dev', '9', 0)), - [(self.testdir + '/dev/objects/9/abc', 'abc'), - (self.testdir + '/dev/objects/9/def', 'def'), - (self.testdir + '/dev/objects/9/012', '012')]) - - def test_yield_hashes_dev_path_fail(self): - self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) - exc = None - try: - list(self.df_mgr.yield_hashes('dev', '9', 0)) - except DiskFileDeviceUnavailable as err: - exc = err - self.assertEqual(str(exc), '') - - def test_yield_hashes_empty(self): - def _listdir(path): - return [] - - with mock.patch('os.listdir', _listdir): - self.assertEqual(list(self.df_mgr.yield_hashes('dev', '9', 0)), []) - - def test_yield_hashes_empty_suffixes(self): - def _listdir(path): - return [] - - with mock.patch('os.listdir', _listdir): - self.assertEqual( - list(self.df_mgr.yield_hashes('dev', '9', 0, - suffixes=['456'])), []) - - def test_yield_hashes(self): - fresh_ts = Timestamp(time() - 10).internal - fresher_ts = Timestamp(time() - 1).internal - - def _listdir(path): - if path.endswith('/dev/objects/9'): - return ['abc', '456', 'def'] - elif path.endswith('/dev/objects/9/abc'): - return ['9373a92d072897b136b3fc06595b4abc'] - elif path.endswith( - '/dev/objects/9/abc/9373a92d072897b136b3fc06595b4abc'): - return [fresh_ts + '.ts'] - elif path.endswith('/dev/objects/9/456'): - return ['9373a92d072897b136b3fc06595b0456', - '9373a92d072897b136b3fc06595b7456'] - elif path.endswith( - '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456'): - return ['1383180000.12345.data'] - elif path.endswith( - '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456'): - return [fresh_ts + '.ts', - fresher_ts + '.data'] - elif path.endswith('/dev/objects/9/def'): - return [] - else: - raise Exception('Unexpected listdir of %r' % path) - - with nested( - mock.patch('os.listdir', _listdir), - mock.patch('os.unlink')): - self.assertEqual( - list(self.df_mgr.yield_hashes('dev', '9', 0)), - [(self.testdir + - '/dev/objects/9/abc/9373a92d072897b136b3fc06595b4abc', - '9373a92d072897b136b3fc06595b4abc', fresh_ts), - (self.testdir + - '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456', - '9373a92d072897b136b3fc06595b0456', '1383180000.12345'), - (self.testdir + - '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456', - '9373a92d072897b136b3fc06595b7456', fresher_ts)]) - - def test_yield_hashes_suffixes(self): - fresh_ts = Timestamp(time() - 10).internal - fresher_ts = Timestamp(time() - 1).internal - - def _listdir(path): - if path.endswith('/dev/objects/9'): - return ['abc', '456', 'def'] - elif path.endswith('/dev/objects/9/abc'): - return ['9373a92d072897b136b3fc06595b4abc'] - elif path.endswith( - '/dev/objects/9/abc/9373a92d072897b136b3fc06595b4abc'): - return [fresh_ts + '.ts'] - elif path.endswith('/dev/objects/9/456'): - return ['9373a92d072897b136b3fc06595b0456', - '9373a92d072897b136b3fc06595b7456'] - elif path.endswith( - '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456'): - return ['1383180000.12345.data'] - elif path.endswith( - '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456'): - return [fresh_ts + '.ts', - fresher_ts + '.data'] - elif path.endswith('/dev/objects/9/def'): - return [] - else: - raise Exception('Unexpected listdir of %r' % path) - - with nested( - mock.patch('os.listdir', _listdir), - mock.patch('os.unlink')): - self.assertEqual( - list(self.df_mgr.yield_hashes( - 'dev', '9', 0, suffixes=['456'])), - [(self.testdir + - '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456', - '9373a92d072897b136b3fc06595b0456', '1383180000.12345'), - (self.testdir + - '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456', - '9373a92d072897b136b3fc06595b7456', fresher_ts)]) - def test_diskfile_names(self): df = self._simple_get_diskfile() self.assertEqual(df.account, 'a') @@ -2259,10 +2976,11 @@ class TestDiskFile(unittest.TestCase): self.assertEqual(str(exc), '') def test_diskfile_timestamp(self): - self._get_open_disk_file(ts='1383181759.12345') + ts = Timestamp(time()) + self._get_open_disk_file(ts=ts.internal) df = self._simple_get_diskfile() with df.open(): - self.assertEqual(df.timestamp, '1383181759.12345') + self.assertEqual(df.timestamp, ts.internal) def test_error_in_hash_cleanup_listdir(self): @@ -2270,16 +2988,16 @@ class TestDiskFile(unittest.TestCase): raise OSError() df = self._get_open_disk_file() + file_count = len(os.listdir(df._datadir)) ts = time() - with mock.patch("swift.obj.diskfile.hash_cleanup_listdir", - mock_hcl): + with mock.patch(self._manager_mock('hash_cleanup_listdir'), mock_hcl): try: df.delete(ts) except OSError: self.fail("OSError raised when it should have been swallowed") exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) - self.assertEquals(len(dl), 2) + self.assertEquals(len(dl), file_count + 1) self.assertTrue(exp_name in set(dl)) def _system_can_zero_copy(self): @@ -2300,7 +3018,6 @@ class TestDiskFile(unittest.TestCase): self.conf['splice'] = 'on' self.conf['keep_cache_size'] = 16384 self.conf['disk_chunk_size'] = 4096 - self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) df = self._get_open_disk_file(fsize=16385) reader = df.reader() @@ -2314,7 +3031,7 @@ class TestDiskFile(unittest.TestCase): def test_zero_copy_turns_off_when_md5_sockets_not_supported(self): if not self._system_can_zero_copy(): raise SkipTest("zero-copy support is missing") - + df_mgr = self.df_router[POLICIES.default] self.conf['splice'] = 'on' with mock.patch('swift.obj.diskfile.get_md5_socket') as mock_md5sock: mock_md5sock.side_effect = IOError( @@ -2323,7 +3040,7 @@ class TestDiskFile(unittest.TestCase): reader = df.reader() self.assertFalse(reader.can_zero_copy_send()) - log_lines = self.df_mgr.logger.get_lines_for_level('warning') + log_lines = df_mgr.logger.get_lines_for_level('warning') self.assert_('MD5 sockets' in log_lines[-1]) def test_tee_to_md5_pipe_length_mismatch(self): @@ -2420,7 +3137,7 @@ class TestDiskFile(unittest.TestCase): def test_create_unlink_cleanup_DiskFileNoSpace(self): # Test cleanup when DiskFileNoSpace() is raised. df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', - 'xyz') + 'xyz', policy=POLICIES.legacy) _m_fallocate = mock.MagicMock(side_effect=OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))) _m_unlink = mock.Mock() @@ -2435,7 +3152,7 @@ class TestDiskFile(unittest.TestCase): self.fail("Expected exception DiskFileNoSpace") self.assertTrue(_m_fallocate.called) self.assertTrue(_m_unlink.called) - self.assert_(len(self.df_mgr.logger.log_dict['exception']) == 0) + self.assertTrue('error' not in self.logger.all_log_lines()) def test_create_unlink_cleanup_renamer_fails(self): # Test cleanup when renamer fails @@ -2462,12 +3179,12 @@ class TestDiskFile(unittest.TestCase): self.assertFalse(writer.put_succeeded) self.assertTrue(_m_renamer.called) self.assertTrue(_m_unlink.called) - self.assert_(len(self.df_mgr.logger.log_dict['exception']) == 0) + self.assertTrue('error' not in self.logger.all_log_lines()) def test_create_unlink_cleanup_logging(self): # Test logging of os.unlink() failures. df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', - 'xyz') + 'xyz', policy=POLICIES.legacy) _m_fallocate = mock.MagicMock(side_effect=OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))) _m_unlink = mock.MagicMock(side_effect=OSError(errno.ENOENT, @@ -2483,8 +3200,1633 @@ class TestDiskFile(unittest.TestCase): self.fail("Expected exception DiskFileNoSpace") self.assertTrue(_m_fallocate.called) self.assertTrue(_m_unlink.called) - self.assert_(self.df_mgr.logger.log_dict['exception'][0][0][0]. - startswith("Error removing tempfile:")) + error_lines = self.logger.get_lines_for_level('error') + for line in error_lines: + self.assertTrue(line.startswith("Error removing tempfile:")) + + +@patch_policies(test_policies) +class TestDiskFile(DiskFileMixin, unittest.TestCase): + + mgr_cls = diskfile.DiskFileManager + + +@patch_policies(with_ec_default=True) +class TestECDiskFile(DiskFileMixin, unittest.TestCase): + + mgr_cls = diskfile.ECDiskFileManager + + def test_commit_raises_DiskFileErrors(self): + scenarios = ((errno.ENOSPC, DiskFileNoSpace), + (errno.EDQUOT, DiskFileNoSpace), + (errno.ENOTDIR, DiskFileError), + (errno.EPERM, DiskFileError)) + + # Check IOErrors from open() is handled + for err_number, expected_exception in scenarios: + io_error = IOError() + io_error.errno = err_number + mock_open = mock.MagicMock(side_effect=io_error) + df = self._simple_get_diskfile(account='a', container='c', + obj='o_%s' % err_number, + policy=POLICIES.default) + timestamp = Timestamp(time()) + with df.create() as writer: + metadata = { + 'ETag': 'bogus_etag', + 'X-Timestamp': timestamp.internal, + 'Content-Length': '0', + } + writer.put(metadata) + with mock.patch('__builtin__.open', mock_open): + self.assertRaises(expected_exception, + writer.commit, + timestamp) + dl = os.listdir(df._datadir) + self.assertEqual(1, len(dl), dl) + rmtree(df._datadir) + + # Check OSError from fsync() is handled + mock_fsync = mock.MagicMock(side_effect=OSError) + df = self._simple_get_diskfile(account='a', container='c', + obj='o_fsync_error') + + timestamp = Timestamp(time()) + with df.create() as writer: + metadata = { + 'ETag': 'bogus_etag', + 'X-Timestamp': timestamp.internal, + 'Content-Length': '0', + } + writer.put(metadata) + with mock.patch('swift.obj.diskfile.fsync', mock_fsync): + self.assertRaises(DiskFileError, + writer.commit, timestamp) + + def test_data_file_has_frag_index(self): + policy = POLICIES.default + for good_value in (0, '0', 2, '2', 14, '14'): + # frag_index set by constructor arg + ts = self.ts().internal + expected = ['%s#%s.data' % (ts, good_value), '%s.durable' % ts] + df = self._get_open_disk_file(ts=ts, policy=policy, + frag_index=good_value) + self.assertEqual(expected, sorted(os.listdir(df._datadir))) + # frag index should be added to object sysmeta + actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index') + self.assertEqual(int(good_value), int(actual)) + + # metadata value overrides the constructor arg + ts = self.ts().internal + expected = ['%s#%s.data' % (ts, good_value), '%s.durable' % ts] + meta = {'X-Object-Sysmeta-Ec-Frag-Index': good_value} + df = self._get_open_disk_file(ts=ts, policy=policy, + frag_index='99', + extra_metadata=meta) + self.assertEqual(expected, sorted(os.listdir(df._datadir))) + actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index') + self.assertEqual(int(good_value), int(actual)) + + # metadata value alone is sufficient + ts = self.ts().internal + expected = ['%s#%s.data' % (ts, good_value), '%s.durable' % ts] + meta = {'X-Object-Sysmeta-Ec-Frag-Index': good_value} + df = self._get_open_disk_file(ts=ts, policy=policy, + frag_index=None, + extra_metadata=meta) + self.assertEqual(expected, sorted(os.listdir(df._datadir))) + actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index') + self.assertEqual(int(good_value), int(actual)) + + def test_sysmeta_frag_index_is_immutable(self): + # the X-Object-Sysmeta-Ec-Frag-Index should *only* be set when + # the .data file is written. + policy = POLICIES.default + orig_frag_index = 14 + # frag_index set by constructor arg + ts = self.ts().internal + expected = ['%s#%s.data' % (ts, orig_frag_index), '%s.durable' % ts] + df = self._get_open_disk_file(ts=ts, policy=policy, obj_name='my_obj', + frag_index=orig_frag_index) + self.assertEqual(expected, sorted(os.listdir(df._datadir))) + # frag index should be added to object sysmeta + actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index') + self.assertEqual(int(orig_frag_index), int(actual)) + + # open the same diskfile with no frag_index passed to constructor + df = self.df_router[policy].get_diskfile( + self.existing_device, 0, 'a', 'c', 'my_obj', policy=policy, + frag_index=None) + df.open() + actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index') + self.assertEqual(int(orig_frag_index), int(actual)) + + # write metadata to a meta file + ts = self.ts().internal + metadata = {'X-Timestamp': ts, + 'X-Object-Meta-Fruit': 'kiwi'} + df.write_metadata(metadata) + # sanity check we did write a meta file + expected.append('%s.meta' % ts) + actual_files = sorted(os.listdir(df._datadir)) + self.assertEqual(expected, actual_files) + + # open the same diskfile, check frag index is unchanged + df = self.df_router[policy].get_diskfile( + self.existing_device, 0, 'a', 'c', 'my_obj', policy=policy, + frag_index=None) + df.open() + # sanity check we have read the meta file + self.assertEqual(ts, df.get_metadata().get('X-Timestamp')) + self.assertEqual('kiwi', df.get_metadata().get('X-Object-Meta-Fruit')) + # check frag index sysmeta is unchanged + actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index') + self.assertEqual(int(orig_frag_index), int(actual)) + + # attempt to overwrite frag index sysmeta + ts = self.ts().internal + metadata = {'X-Timestamp': ts, + 'X-Object-Sysmeta-Ec-Frag-Index': 99, + 'X-Object-Meta-Fruit': 'apple'} + df.write_metadata(metadata) + + # open the same diskfile, check frag index is unchanged + df = self.df_router[policy].get_diskfile( + self.existing_device, 0, 'a', 'c', 'my_obj', policy=policy, + frag_index=None) + df.open() + # sanity check we have read the meta file + self.assertEqual(ts, df.get_metadata().get('X-Timestamp')) + self.assertEqual('apple', df.get_metadata().get('X-Object-Meta-Fruit')) + actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index') + self.assertEqual(int(orig_frag_index), int(actual)) + + def test_data_file_errors_bad_frag_index(self): + policy = POLICIES.default + df_mgr = self.df_router[policy] + for bad_value in ('foo', '-2', -2, '3.14', 3.14): + # check that bad frag_index set by constructor arg raises error + # as soon as diskfile is constructed, before data is written + self.assertRaises(DiskFileError, self._simple_get_diskfile, + policy=policy, frag_index=bad_value) + + # bad frag_index set by metadata value + # (drive-by check that it is ok for constructor arg to be None) + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', 'o', + policy=policy, frag_index=None) + ts = self.ts() + meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value, + 'X-Timestamp': ts.internal, + 'Content-Length': 0, + 'Etag': EMPTY_ETAG, + 'Content-Type': 'plain/text'} + with df.create() as writer: + try: + writer.put(meta) + self.fail('Expected DiskFileError for frag_index %s' + % bad_value) + except DiskFileError: + pass + + # bad frag_index set by metadata value overrides ok constructor arg + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', 'o', + policy=policy, frag_index=2) + ts = self.ts() + meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value, + 'X-Timestamp': ts.internal, + 'Content-Length': 0, + 'Etag': EMPTY_ETAG, + 'Content-Type': 'plain/text'} + with df.create() as writer: + try: + writer.put(meta) + self.fail('Expected DiskFileError for frag_index %s' + % bad_value) + except DiskFileError: + pass + + def test_purge_one_fragment_index(self): + ts = self.ts() + for frag_index in (1, 2): + df = self._simple_get_diskfile(frag_index=frag_index) + with df.create() as writer: + data = 'test data' + writer.write(data) + metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': ts.internal, + 'Content-Length': len(data), + } + writer.put(metadata) + writer.commit(ts) + + # sanity + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '#1.data', + ts.internal + '#2.data', + ts.internal + '.durable', + ]) + df.purge(ts, 2) + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '#1.data', + ts.internal + '.durable', + ]) + + def test_purge_last_fragment_index(self): + ts = self.ts() + frag_index = 0 + df = self._simple_get_diskfile(frag_index=frag_index) + with df.create() as writer: + data = 'test data' + writer.write(data) + metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': ts.internal, + 'Content-Length': len(data), + } + writer.put(metadata) + writer.commit(ts) + + # sanity + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '#0.data', + ts.internal + '.durable', + ]) + df.purge(ts, 0) + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '.durable', + ]) + + def test_purge_non_existant_fragment_index(self): + ts = self.ts() + frag_index = 7 + df = self._simple_get_diskfile(frag_index=frag_index) + with df.create() as writer: + data = 'test data' + writer.write(data) + metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': ts.internal, + 'Content-Length': len(data), + } + writer.put(metadata) + writer.commit(ts) + + # sanity + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '#7.data', + ts.internal + '.durable', + ]) + df.purge(ts, 3) + # no effect + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '#7.data', + ts.internal + '.durable', + ]) + + def test_purge_old_timestamp_frag_index(self): + old_ts = self.ts() + ts = self.ts() + frag_index = 1 + df = self._simple_get_diskfile(frag_index=frag_index) + with df.create() as writer: + data = 'test data' + writer.write(data) + metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': ts.internal, + 'Content-Length': len(data), + } + writer.put(metadata) + writer.commit(ts) + + # sanity + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '#1.data', + ts.internal + '.durable', + ]) + df.purge(old_ts, 1) + # no effect + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '#1.data', + ts.internal + '.durable', + ]) + + def test_purge_tombstone(self): + ts = self.ts() + df = self._simple_get_diskfile(frag_index=3) + df.delete(ts) + + # sanity + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '.ts', + ]) + df.purge(ts, 3) + self.assertEqual(sorted(os.listdir(df._datadir)), []) + + def test_purge_old_tombstone(self): + old_ts = self.ts() + ts = self.ts() + df = self._simple_get_diskfile(frag_index=5) + df.delete(ts) + + # sanity + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '.ts', + ]) + df.purge(old_ts, 5) + # no effect + self.assertEqual(sorted(os.listdir(df._datadir)), [ + ts.internal + '.ts', + ]) + + def test_purge_already_removed(self): + df = self._simple_get_diskfile(frag_index=6) + + df.purge(self.ts(), 6) # no errors + + # sanity + os.makedirs(df._datadir) + self.assertEqual(sorted(os.listdir(df._datadir)), []) + df.purge(self.ts(), 6) + # no effect + self.assertEqual(sorted(os.listdir(df._datadir)), []) + + def test_open_most_recent_durable(self): + policy = POLICIES.default + df_mgr = self.df_router[policy] + + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + + ts = self.ts() + with df.create() as writer: + data = 'test data' + writer.write(data) + metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': ts.internal, + 'Content-Length': len(data), + 'X-Object-Sysmeta-Ec-Frag-Index': 3, + } + writer.put(metadata) + writer.commit(ts) + + # add some .meta stuff + extra_meta = { + 'X-Object-Meta-Foo': 'Bar', + 'X-Timestamp': self.ts().internal, + } + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + df.write_metadata(extra_meta) + + # sanity + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + metadata.update(extra_meta) + self.assertEqual(metadata, df.read_metadata()) + + # add a newer datafile + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + ts = self.ts() + with df.create() as writer: + data = 'test data' + writer.write(data) + new_metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': ts.internal, + 'Content-Length': len(data), + 'X-Object-Sysmeta-Ec-Frag-Index': 3, + } + writer.put(new_metadata) + # N.B. don't make it durable + + # and we still get the old metadata (same as if no .data!) + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + self.assertEqual(metadata, df.read_metadata()) + + def test_open_most_recent_missing_durable(self): + policy = POLICIES.default + df_mgr = self.df_router[policy] + + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + + self.assertRaises(DiskFileNotExist, df.read_metadata) + + # now create a datafile missing durable + ts = self.ts() + with df.create() as writer: + data = 'test data' + writer.write(data) + new_metadata = { + 'ETag': md5(data).hexdigest(), + 'X-Timestamp': ts.internal, + 'Content-Length': len(data), + 'X-Object-Sysmeta-Ec-Frag-Index': 3, + } + writer.put(new_metadata) + # N.B. don't make it durable + + # add some .meta stuff + extra_meta = { + 'X-Object-Meta-Foo': 'Bar', + 'X-Timestamp': self.ts().internal, + } + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + df.write_metadata(extra_meta) + + # we still get the DiskFileNotExist (same as if no .data!) + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy, + frag_index=3) + self.assertRaises(DiskFileNotExist, df.read_metadata) + + # sanity, withtout the frag_index kwarg + df = df_mgr.get_diskfile(self.existing_device, '0', + 'a', 'c', 'o', policy=policy) + self.assertRaises(DiskFileNotExist, df.read_metadata) + + +@patch_policies(with_ec_default=True) +class TestSuffixHashes(unittest.TestCase): + """ + This tests all things related to hashing suffixes and therefore + there's also few test methods for hash_cleanup_listdir as well + (because it's used by hash_suffix). + + The public interface to suffix hashing is on the Manager:: + + * hash_cleanup_listdir(hsh_path) + * get_hashes(device, partition, suffixes, policy) + * invalidate_hash(suffix_dir) + + The Manager.get_hashes method (used by the REPLICATION verb) + calls Manager._get_hashes (which may be an alias to the module + method get_hashes), which calls hash_suffix, which calls + hash_cleanup_listdir. + + Outside of that, hash_cleanup_listdir and invalidate_hash are + used mostly after writing new files via PUT or DELETE. + + Test methods are organized by:: + + * hash_cleanup_listdir tests - behaviors + * hash_cleanup_listdir tests - error handling + * invalidate_hash tests - behavior + * invalidate_hash tests - error handling + * get_hashes tests - hash_suffix behaviors + * get_hashes tests - hash_suffix error handling + * get_hashes tests - behaviors + * get_hashes tests - error handling + + """ + + def setUp(self): + self.testdir = tempfile.mkdtemp() + self.logger = debug_logger('suffix-hash-test') + self.devices = os.path.join(self.testdir, 'node') + os.mkdir(self.devices) + self.existing_device = 'sda1' + os.mkdir(os.path.join(self.devices, self.existing_device)) + self.conf = { + 'swift_dir': self.testdir, + 'devices': self.devices, + 'mount_check': False, + } + self.df_router = diskfile.DiskFileRouter(self.conf, self.logger) + self._ts_iter = (Timestamp(t) for t in + itertools.count(int(time()))) + self.policy = None + + def ts(self): + """ + Timestamps - forever. + """ + return next(self._ts_iter) + + def fname_to_ts_hash(self, fname): + """ + EC datafiles are only hashed by their timestamp + """ + return md5(fname.split('#', 1)[0]).hexdigest() + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def iter_policies(self): + for policy in POLICIES: + self.policy = policy + yield policy + + def assertEqual(self, *args): + try: + unittest.TestCase.assertEqual(self, *args) + except AssertionError as err: + if not self.policy: + raise + policy_trailer = '\n\n... for policy %r' % self.policy + raise AssertionError(str(err) + policy_trailer) + + def _datafilename(self, timestamp, policy, frag_index=None): + if frag_index is None: + frag_index = randint(0, 9) + filename = timestamp.internal + if policy.policy_type == EC_POLICY: + filename += '#%d' % frag_index + filename += '.data' + return filename + + def check_hash_cleanup_listdir(self, policy, input_files, output_files): + orig_unlink = os.unlink + file_list = list(input_files) + + def mock_listdir(path): + return list(file_list) + + def mock_unlink(path): + # timestamp 1 is a special tag to pretend a file disappeared + # between the listdir and unlink. + if '/0000000001.00000.' in path: + # Using actual os.unlink for a non-existent name to reproduce + # exactly what OSError it raises in order to prove that + # common.utils.remove_file is squelching the error - but any + # OSError would do. + orig_unlink(uuid.uuid4().hex) + file_list.remove(os.path.basename(path)) + + df_mgr = self.df_router[policy] + with unit_mock({'os.listdir': mock_listdir, 'os.unlink': mock_unlink}): + if isinstance(output_files, Exception): + path = os.path.join(self.testdir, 'does-not-matter') + self.assertRaises(output_files.__class__, + df_mgr.hash_cleanup_listdir, path) + return + files = df_mgr.hash_cleanup_listdir('/whatever') + self.assertEquals(files, output_files) + + # hash_cleanup_listdir tests - behaviors + + def test_hash_cleanup_listdir_purge_data_newer_ts(self): + for policy in self.iter_policies(): + # purge .data if there's a newer .ts + file1 = self._datafilename(self.ts(), policy) + file2 = self.ts().internal + '.ts' + file_list = [file1, file2] + self.check_hash_cleanup_listdir(policy, file_list, [file2]) + + def test_hash_cleanup_listdir_purge_expired_ts(self): + for policy in self.iter_policies(): + # purge older .ts files if there's a newer .data + file1 = self.ts().internal + '.ts' + file2 = self.ts().internal + '.ts' + timestamp = self.ts() + file3 = self._datafilename(timestamp, policy) + file_list = [file1, file2, file3] + expected = { + # no durable datafile means you can't get rid of the + # latest tombstone even if datafile is newer + EC_POLICY: [file3, file2], + REPL_POLICY: [file3], + }[policy.policy_type] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_purge_ts_newer_data(self): + for policy in self.iter_policies(): + # purge .ts if there's a newer .data + file1 = self.ts().internal + '.ts' + timestamp = self.ts() + file2 = self._datafilename(timestamp, policy) + file_list = [file1, file2] + if policy.policy_type == EC_POLICY: + durable_file = timestamp.internal + '.durable' + file_list.append(durable_file) + expected = { + EC_POLICY: [durable_file, file2], + REPL_POLICY: [file2], + }[policy.policy_type] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_purge_older_ts(self): + for policy in self.iter_policies(): + file1 = self.ts().internal + '.ts' + file2 = self.ts().internal + '.ts' + file3 = self._datafilename(self.ts(), policy) + file4 = self.ts().internal + '.meta' + expected = { + # no durable means we can only throw out things before + # the latest tombstone + EC_POLICY: [file4, file3, file2], + # keep .meta and .data and purge all .ts files + REPL_POLICY: [file4, file3], + }[policy.policy_type] + file_list = [file1, file2, file3, file4] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_keep_meta_data_purge_ts(self): + for policy in self.iter_policies(): + file1 = self.ts().internal + '.ts' + file2 = self.ts().internal + '.ts' + timestamp = self.ts() + file3 = self._datafilename(timestamp, policy) + file_list = [file1, file2, file3] + if policy.policy_type == EC_POLICY: + durable_filename = timestamp.internal + '.durable' + file_list.append(durable_filename) + file4 = self.ts().internal + '.meta' + file_list.append(file4) + # keep .meta and .data if meta newer than data and purge .ts + expected = { + EC_POLICY: [file4, durable_filename, file3], + REPL_POLICY: [file4, file3], + }[policy.policy_type] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_keep_one_ts(self): + for policy in self.iter_policies(): + file1, file2, file3 = [self.ts().internal + '.ts' + for i in range(3)] + file_list = [file1, file2, file3] + # keep only latest of multiple .ts files + self.check_hash_cleanup_listdir(policy, file_list, [file3]) + + def test_hash_cleanup_listdir_multi_data_file(self): + for policy in self.iter_policies(): + file1 = self._datafilename(self.ts(), policy, 1) + file2 = self._datafilename(self.ts(), policy, 2) + file3 = self._datafilename(self.ts(), policy, 3) + expected = { + # keep all non-durable datafiles + EC_POLICY: [file3, file2, file1], + # keep only latest of multiple .data files + REPL_POLICY: [file3] + }[policy.policy_type] + file_list = [file1, file2, file3] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_keeps_one_datafile(self): + for policy in self.iter_policies(): + timestamps = [self.ts() for i in range(3)] + file1 = self._datafilename(timestamps[0], policy, 1) + file2 = self._datafilename(timestamps[1], policy, 2) + file3 = self._datafilename(timestamps[2], policy, 3) + file_list = [file1, file2, file3] + if policy.policy_type == EC_POLICY: + for t in timestamps: + file_list.append(t.internal + '.durable') + latest_durable = file_list[-1] + expected = { + # keep latest durable and datafile + EC_POLICY: [latest_durable, file3], + # keep only latest of multiple .data files + REPL_POLICY: [file3] + }[policy.policy_type] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_keep_one_meta(self): + for policy in self.iter_policies(): + # keep only latest of multiple .meta files + t_data = self.ts() + file1 = self._datafilename(t_data, policy) + file2, file3 = [self.ts().internal + '.meta' for i in range(2)] + file_list = [file1, file2, file3] + if policy.policy_type == EC_POLICY: + durable_file = t_data.internal + '.durable' + file_list.append(durable_file) + expected = { + EC_POLICY: [file3, durable_file, file1], + REPL_POLICY: [file3, file1] + }[policy.policy_type] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_only_meta(self): + for policy in self.iter_policies(): + file1, file2 = [self.ts().internal + '.meta' for i in range(2)] + file_list = [file1, file2] + if policy.policy_type == EC_POLICY: + # EC policy does tolerate only .meta's in dir when cleaning up + expected = [file2] + else: + # the get_ondisk_files contract validation doesn't allow a + # directory with only .meta files + expected = AssertionError() + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_ignore_orphaned_ts(self): + for policy in self.iter_policies(): + # A more recent orphaned .meta file will prevent old .ts files + # from being cleaned up otherwise + file1, file2 = [self.ts().internal + '.ts' for i in range(2)] + file3 = self.ts().internal + '.meta' + file_list = [file1, file2, file3] + self.check_hash_cleanup_listdir(policy, file_list, [file3, file2]) + + def test_hash_cleanup_listdir_purge_old_data_only(self): + for policy in self.iter_policies(): + # Oldest .data will be purge, .meta and .ts won't be touched + file1 = self._datafilename(self.ts(), policy) + file2 = self.ts().internal + '.ts' + file3 = self.ts().internal + '.meta' + file_list = [file1, file2, file3] + self.check_hash_cleanup_listdir(policy, file_list, [file3, file2]) + + def test_hash_cleanup_listdir_purge_old_ts(self): + for policy in self.iter_policies(): + # A single old .ts file will be removed + old_float = time() - (diskfile.ONE_WEEK + 1) + file1 = Timestamp(old_float).internal + '.ts' + file_list = [file1] + self.check_hash_cleanup_listdir(policy, file_list, []) + + def test_hash_cleanup_listdir_meta_keeps_old_ts(self): + for policy in self.iter_policies(): + old_float = time() - (diskfile.ONE_WEEK + 1) + file1 = Timestamp(old_float).internal + '.ts' + file2 = Timestamp(time() + 2).internal + '.meta' + file_list = [file1, file2] + if policy.policy_type == EC_POLICY: + # EC will clean up old .ts despite a .meta + expected = [file2] + else: + # An orphaned .meta will not clean up a very old .ts + expected = [file2, file1] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_keep_single_old_data(self): + for policy in self.iter_policies(): + old_float = time() - (diskfile.ONE_WEEK + 1) + file1 = self._datafilename(Timestamp(old_float), policy) + file_list = [file1] + if policy.policy_type == EC_POLICY: + # for EC an isolated old .data file is removed, its useless + # without a .durable + expected = [] + else: + # A single old .data file will not be removed + expected = file_list + self.check_hash_cleanup_listdir(policy, file_list, expected) + + def test_hash_cleanup_listdir_drops_isolated_durable(self): + for policy in self.iter_policies(): + if policy.policy_type == EC_POLICY: + file1 = Timestamp(time()).internal + '.durable' + file_list = [file1] + self.check_hash_cleanup_listdir(policy, file_list, []) + + def test_hash_cleanup_listdir_keep_single_old_meta(self): + for policy in self.iter_policies(): + # A single old .meta file will not be removed + old_float = time() - (diskfile.ONE_WEEK + 1) + file1 = Timestamp(old_float).internal + '.meta' + file_list = [file1] + self.check_hash_cleanup_listdir(policy, file_list, [file1]) + + # hash_cleanup_listdir tests - error handling + + def test_hash_cleanup_listdir_hsh_path_enoent(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # common.utils.listdir *completely* mutes ENOENT + path = os.path.join(self.testdir, 'does-not-exist') + self.assertEqual(df_mgr.hash_cleanup_listdir(path), []) + + def test_hash_cleanup_listdir_hsh_path_other_oserror(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + with mock.patch('os.listdir') as mock_listdir: + mock_listdir.side_effect = OSError('kaboom!') + # but it will raise other OSErrors + path = os.path.join(self.testdir, 'does-not-matter') + self.assertRaises(OSError, df_mgr.hash_cleanup_listdir, + path) + + def test_hash_cleanup_listdir_reclaim_tombstone_remove_file_error(self): + for policy in self.iter_policies(): + # Timestamp 1 makes the check routine pretend the file + # disappeared after listdir before unlink. + file1 = '0000000001.00000.ts' + file_list = [file1] + self.check_hash_cleanup_listdir(policy, file_list, []) + + def test_hash_cleanup_listdir_older_remove_file_error(self): + for policy in self.iter_policies(): + # Timestamp 1 makes the check routine pretend the file + # disappeared after listdir before unlink. + file1 = self._datafilename(Timestamp(1), policy) + file2 = '0000000002.00000.ts' + file_list = [file1, file2] + if policy.policy_type == EC_POLICY: + # the .ts gets reclaimed up despite failed .data delete + expected = [] + else: + # the .ts isn't reclaimed because there were two files in dir + expected = [file2] + self.check_hash_cleanup_listdir(policy, file_list, expected) + + # invalidate_hash tests - behavior + + def test_invalidate_hash_file_does_not_exist(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o', + policy=policy) + suffix_dir = os.path.dirname(df._datadir) + part_path = os.path.join(self.devices, 'sda1', + diskfile.get_data_dir(policy), '0') + hashes_file = os.path.join(part_path, diskfile.HASH_FILE) + self.assertFalse(os.path.exists(hashes_file)) # sanity + with mock.patch('swift.obj.diskfile.lock_path') as mock_lock: + df_mgr.invalidate_hash(suffix_dir) + self.assertFalse(mock_lock.called) + # does not create file + self.assertFalse(os.path.exists(hashes_file)) + + def test_invalidate_hash_file_exists(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # create something to hash + df = df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o', + policy=policy) + df.delete(self.ts()) + suffix_dir = os.path.dirname(df._datadir) + suffix = os.path.basename(suffix_dir) + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + self.assertTrue(suffix in hashes) # sanity + # sanity check hashes file + part_path = os.path.join(self.devices, 'sda1', + diskfile.get_data_dir(policy), '0') + hashes_file = os.path.join(part_path, diskfile.HASH_FILE) + with open(hashes_file, 'rb') as f: + self.assertEqual(hashes, pickle.load(f)) + # invalidate the hash + with mock.patch('swift.obj.diskfile.lock_path') as mock_lock: + df_mgr.invalidate_hash(suffix_dir) + self.assertTrue(mock_lock.called) + with open(hashes_file, 'rb') as f: + self.assertEqual({suffix: None}, pickle.load(f)) + + # invalidate_hash tests - error handling + + def test_invalidate_hash_bad_pickle(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # make some valid data + df = df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o', + policy=policy) + suffix_dir = os.path.dirname(df._datadir) + suffix = os.path.basename(suffix_dir) + df.delete(self.ts()) + # sanity check hashes file + part_path = os.path.join(self.devices, 'sda1', + diskfile.get_data_dir(policy), '0') + hashes_file = os.path.join(part_path, diskfile.HASH_FILE) + self.assertFalse(os.path.exists(hashes_file)) + # write some garbage in hashes file + with open(hashes_file, 'w') as f: + f.write('asdf') + # invalidate_hash silently *NOT* repair invalid data + df_mgr.invalidate_hash(suffix_dir) + with open(hashes_file) as f: + self.assertEqual(f.read(), 'asdf') + # ... but get_hashes will + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + self.assertTrue(suffix in hashes) + + # get_hashes tests - hash_suffix behaviors + + def test_hash_suffix_one_tombstone(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile( + 'sda1', '0', 'a', 'c', 'o', policy=policy) + suffix = os.path.basename(os.path.dirname(df._datadir)) + # write a tombstone + timestamp = self.ts() + df.delete(timestamp) + tombstone_hash = md5(timestamp.internal + '.ts').hexdigest() + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + expected = { + REPL_POLICY: {suffix: tombstone_hash}, + EC_POLICY: {suffix: { + # fi is None here because we have a tombstone + None: tombstone_hash}}, + }[policy.policy_type] + self.assertEqual(hashes, expected) + + def test_hash_suffix_one_reclaim_tombstone(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile( + 'sda1', '0', 'a', 'c', 'o', policy=policy) + suffix = os.path.basename(os.path.dirname(df._datadir)) + # scale back this tests manager's reclaim age a bit + df_mgr.reclaim_age = 1000 + # write a tombstone that's just a *little* older + old_time = time() - 1001 + timestamp = Timestamp(old_time) + df.delete(timestamp.internal) + tombstone_hash = md5(timestamp.internal + '.ts').hexdigest() + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + expected = { + # repl is broken, it doesn't use self.reclaim_age + REPL_POLICY: tombstone_hash, + EC_POLICY: {}, + }[policy.policy_type] + self.assertEqual(hashes, {suffix: expected}) + + def test_hash_suffix_one_datafile(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile( + 'sda1', '0', 'a', 'c', 'o', policy=policy, frag_index=7) + suffix = os.path.basename(os.path.dirname(df._datadir)) + # write a datafile + timestamp = self.ts() + with df.create() as writer: + test_data = 'test file' + writer.write(test_data) + metadata = { + 'X-Timestamp': timestamp.internal, + 'ETag': md5(test_data).hexdigest(), + 'Content-Length': len(test_data), + } + writer.put(metadata) + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + datafile_hash = md5({ + EC_POLICY: timestamp.internal, + REPL_POLICY: timestamp.internal + '.data', + }[policy.policy_type]).hexdigest() + expected = { + REPL_POLICY: {suffix: datafile_hash}, + EC_POLICY: {suffix: { + # because there's no .durable file, we have no hash for + # the None key - only the frag index for the data file + 7: datafile_hash}}, + }[policy.policy_type] + msg = 'expected %r != %r for policy %r' % ( + expected, hashes, policy) + self.assertEqual(hashes, expected, msg) + + def test_hash_suffix_multi_file_ends_in_tombstone(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o', policy=policy, + frag_index=4) + suffix = os.path.basename(os.path.dirname(df._datadir)) + mkdirs(df._datadir) + now = time() + # go behind the scenes and setup a bunch of weird file names + for tdiff in [500, 100, 10, 1]: + for suff in ['.meta', '.data', '.ts']: + timestamp = Timestamp(now - tdiff) + filename = timestamp.internal + if policy.policy_type == EC_POLICY and suff == '.data': + filename += '#%s' % df._frag_index + filename += suff + open(os.path.join(df._datadir, filename), 'w').close() + tombstone_hash = md5(filename).hexdigest() + # call get_hashes and it should clean things up + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + expected = { + REPL_POLICY: {suffix: tombstone_hash}, + EC_POLICY: {suffix: { + # fi is None here because we have a tombstone + None: tombstone_hash}}, + }[policy.policy_type] + self.assertEqual(hashes, expected) + # only the tombstone should be left + found_files = os.listdir(df._datadir) + self.assertEqual(found_files, [filename]) + + def test_hash_suffix_multi_file_ends_in_datafile(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o', policy=policy, + frag_index=4) + suffix = os.path.basename(os.path.dirname(df._datadir)) + mkdirs(df._datadir) + now = time() + timestamp = None + # go behind the scenes and setup a bunch of weird file names + for tdiff in [500, 100, 10, 1]: + suffs = ['.meta', '.data'] + if tdiff > 50: + suffs.append('.ts') + if policy.policy_type == EC_POLICY: + suffs.append('.durable') + for suff in suffs: + timestamp = Timestamp(now - tdiff) + filename = timestamp.internal + if policy.policy_type == EC_POLICY and suff == '.data': + filename += '#%s' % df._frag_index + filename += suff + open(os.path.join(df._datadir, filename), 'w').close() + # call get_hashes and it should clean things up + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + data_filename = timestamp.internal + if policy.policy_type == EC_POLICY: + data_filename += '#%s' % df._frag_index + data_filename += '.data' + metadata_filename = timestamp.internal + '.meta' + durable_filename = timestamp.internal + '.durable' + if policy.policy_type == EC_POLICY: + hasher = md5() + hasher.update(metadata_filename) + hasher.update(durable_filename) + expected = { + suffix: { + # metadata & durable updates are hashed separately + None: hasher.hexdigest(), + 4: self.fname_to_ts_hash(data_filename), + } + } + expected_files = [data_filename, durable_filename, + metadata_filename] + elif policy.policy_type == REPL_POLICY: + hasher = md5() + hasher.update(metadata_filename) + hasher.update(data_filename) + expected = {suffix: hasher.hexdigest()} + expected_files = [data_filename, metadata_filename] + else: + self.fail('unknown policy type %r' % policy.policy_type) + msg = 'expected %r != %r for policy %r' % ( + expected, hashes, policy) + self.assertEqual(hashes, expected, msg) + # only the meta and data should be left + self.assertEqual(sorted(os.listdir(df._datadir)), + sorted(expected_files)) + + def test_hash_suffix_removes_empty_hashdir_and_suffix(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o', + policy=policy, frag_index=2) + os.makedirs(df._datadir) + self.assertTrue(os.path.exists(df._datadir)) # sanity + df_mgr.get_hashes('sda1', '0', [], policy) + suffix_dir = os.path.dirname(df._datadir) + self.assertFalse(os.path.exists(suffix_dir)) + + def test_hash_suffix_removes_empty_hashdirs_in_valid_suffix(self): + paths, suffix = find_paths_with_matching_suffixes(needed_matches=3, + needed_suffixes=0) + matching_paths = paths.pop(suffix) + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile('sda1', '0', *matching_paths[0], + policy=policy, frag_index=2) + # create a real, valid hsh_path + df.delete(Timestamp(time())) + # and a couple of empty hsh_paths + empty_hsh_paths = [] + for path in matching_paths[1:]: + fake_df = df_mgr.get_diskfile('sda1', '0', *path, + policy=policy) + os.makedirs(fake_df._datadir) + empty_hsh_paths.append(fake_df._datadir) + for hsh_path in empty_hsh_paths: + self.assertTrue(os.path.exists(hsh_path)) # sanity + # get_hashes will cleanup empty hsh_path and leave valid one + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + self.assertTrue(suffix in hashes) + self.assertTrue(os.path.exists(df._datadir)) + for hsh_path in empty_hsh_paths: + self.assertFalse(os.path.exists(hsh_path)) + + # get_hashes tests - hash_suffix error handling + + def test_hash_suffix_listdir_enotdir(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + suffix = '123' + suffix_path = os.path.join(self.devices, 'sda1', + diskfile.get_data_dir(policy), '0', + suffix) + os.makedirs(suffix_path) + self.assertTrue(os.path.exists(suffix_path)) # sanity + hashes = df_mgr.get_hashes('sda1', '0', [suffix], policy) + # suffix dir cleaned up by get_hashes + self.assertFalse(os.path.exists(suffix_path)) + expected = { + EC_POLICY: {'123': {}}, + REPL_POLICY: {'123': EMPTY_ETAG}, + }[policy.policy_type] + msg = 'expected %r != %r for policy %r' % (expected, hashes, + policy) + self.assertEqual(hashes, expected, msg) + + # now make the suffix path a file + open(suffix_path, 'w').close() + hashes = df_mgr.get_hashes('sda1', '0', [suffix], policy) + expected = {} + msg = 'expected %r != %r for policy %r' % (expected, hashes, + policy) + self.assertEqual(hashes, expected, msg) + + def test_hash_suffix_listdir_enoent(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + orig_listdir = os.listdir + listdir_calls = [] + + def mock_listdir(path): + success = False + try: + rv = orig_listdir(path) + success = True + return rv + finally: + listdir_calls.append((path, success)) + + with mock.patch('swift.obj.diskfile.os.listdir', + mock_listdir): + # recalc always forces hash_suffix even if the suffix + # does not exist! + df_mgr.get_hashes('sda1', '0', ['123'], policy) + + part_path = os.path.join(self.devices, 'sda1', + diskfile.get_data_dir(policy), '0') + + self.assertEqual(listdir_calls, [ + # part path gets created automatically + (part_path, True), + # this one blows up + (os.path.join(part_path, '123'), False), + ]) + + def test_hash_suffix_hash_cleanup_listdir_enotdir_quarantined(self): + for policy in self.iter_policies(): + df = self.df_router[policy].get_diskfile( + self.existing_device, '0', 'a', 'c', 'o', policy=policy) + # make the suffix directory + suffix_path = os.path.dirname(df._datadir) + os.makedirs(suffix_path) + suffix = os.path.basename(suffix_path) + + # make the df hash path a file + open(df._datadir, 'wb').close() + df_mgr = self.df_router[policy] + hashes = df_mgr.get_hashes(self.existing_device, '0', [suffix], + policy) + expected = { + REPL_POLICY: {suffix: EMPTY_ETAG}, + EC_POLICY: {suffix: {}}, + }[policy.policy_type] + self.assertEqual(hashes, expected) + # and hash path is quarantined + self.assertFalse(os.path.exists(df._datadir)) + # each device a quarantined directory + quarantine_base = os.path.join(self.devices, + self.existing_device, 'quarantined') + # the quarantine path is... + quarantine_path = os.path.join( + quarantine_base, # quarantine root + diskfile.get_data_dir(policy), # per-policy data dir + suffix, # first dir from which quarantined file was removed + os.path.basename(df._datadir) # name of quarantined file + ) + self.assertTrue(os.path.exists(quarantine_path)) + + def test_hash_suffix_hash_cleanup_listdir_other_oserror(self): + for policy in self.iter_policies(): + timestamp = self.ts() + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', + 'o', policy=policy, + frag_index=7) + suffix = os.path.basename(os.path.dirname(df._datadir)) + with df.create() as writer: + test_data = 'test_data' + writer.write(test_data) + metadata = { + 'X-Timestamp': timestamp.internal, + 'ETag': md5(test_data).hexdigest(), + 'Content-Length': len(test_data), + } + writer.put(metadata) + + orig_os_listdir = os.listdir + listdir_calls = [] + + part_path = os.path.join(self.devices, self.existing_device, + diskfile.get_data_dir(policy), '0') + suffix_path = os.path.join(part_path, suffix) + datadir_path = os.path.join(suffix_path, hash_path('a', 'c', 'o')) + + def mock_os_listdir(path): + listdir_calls.append(path) + if path == datadir_path: + # we want the part and suffix listdir calls to pass and + # make the hash_cleanup_listdir raise an exception + raise OSError(errno.EACCES, os.strerror(errno.EACCES)) + return orig_os_listdir(path) + + with mock.patch('os.listdir', mock_os_listdir): + hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + + self.assertEqual(listdir_calls, [ + part_path, + suffix_path, + datadir_path, + ]) + expected = {suffix: None} + msg = 'expected %r != %r for policy %r' % ( + expected, hashes, policy) + self.assertEqual(hashes, expected, msg) + + def test_hash_suffix_rmdir_hsh_path_oserror(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # make an empty hsh_path to be removed + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', + 'o', policy=policy) + os.makedirs(df._datadir) + suffix = os.path.basename(os.path.dirname(df._datadir)) + with mock.patch('os.rmdir', side_effect=OSError()): + hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + expected = { + EC_POLICY: {}, + REPL_POLICY: md5().hexdigest(), + }[policy.policy_type] + self.assertEqual(hashes, {suffix: expected}) + self.assertTrue(os.path.exists(df._datadir)) + + def test_hash_suffix_rmdir_suffix_oserror(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # make an empty hsh_path to be removed + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', + 'o', policy=policy) + os.makedirs(df._datadir) + suffix_path = os.path.dirname(df._datadir) + suffix = os.path.basename(suffix_path) + + captured_paths = [] + + def mock_rmdir(path): + captured_paths.append(path) + if path == suffix_path: + raise OSError('kaboom!') + + with mock.patch('os.rmdir', mock_rmdir): + hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + expected = { + EC_POLICY: {}, + REPL_POLICY: md5().hexdigest(), + }[policy.policy_type] + self.assertEqual(hashes, {suffix: expected}) + self.assertTrue(os.path.exists(suffix_path)) + self.assertEqual([ + df._datadir, + suffix_path, + ], captured_paths) + + # get_hashes tests - behaviors + + def test_get_hashes_creates_partition_and_pkl(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + self.assertEqual(hashes, {}) + part_path = os.path.join( + self.devices, 'sda1', diskfile.get_data_dir(policy), '0') + self.assertTrue(os.path.exists(part_path)) + hashes_file = os.path.join(part_path, + diskfile.HASH_FILE) + self.assertTrue(os.path.exists(hashes_file)) + + # and double check the hashes + new_hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + self.assertEqual(hashes, new_hashes) + + def test_get_hashes_new_pkl_finds_new_suffix_dirs(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + part_path = os.path.join( + self.devices, self.existing_device, + diskfile.get_data_dir(policy), '0') + hashes_file = os.path.join(part_path, + diskfile.HASH_FILE) + # add something to find + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', + 'o', policy=policy, frag_index=4) + timestamp = self.ts() + df.delete(timestamp) + suffix = os.path.basename(os.path.dirname(df._datadir)) + # get_hashes will find the untracked suffix dir + self.assertFalse(os.path.exists(hashes_file)) # sanity + hashes = df_mgr.get_hashes(self.existing_device, '0', [], policy) + self.assertTrue(suffix in hashes) + # ... and create a hashes pickle for it + self.assertTrue(os.path.exists(hashes_file)) + + def test_get_hashes_old_pickle_does_not_find_new_suffix_dirs(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # create a empty stale pickle + part_path = os.path.join( + self.devices, 'sda1', diskfile.get_data_dir(policy), '0') + hashes_file = os.path.join(part_path, + diskfile.HASH_FILE) + hashes = df_mgr.get_hashes(self.existing_device, '0', [], policy) + self.assertEqual(hashes, {}) + self.assertTrue(os.path.exists(hashes_file)) # sanity + # add something to find + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', 'o', + policy=policy, frag_index=4) + os.makedirs(df._datadir) + filename = Timestamp(time()).internal + '.ts' + open(os.path.join(df._datadir, filename), 'w').close() + suffix = os.path.basename(os.path.dirname(df._datadir)) + # but get_hashes has no reason to find it (because we didn't + # call invalidate_hash) + new_hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + self.assertEqual(new_hashes, hashes) + # ... unless remote end asks for a recalc + hashes = df_mgr.get_hashes(self.existing_device, '0', [suffix], + policy) + self.assertTrue(suffix in hashes) + + def test_get_hashes_does_not_rehash_known_suffix_dirs(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', + 'o', policy=policy, frag_index=4) + suffix = os.path.basename(os.path.dirname(df._datadir)) + timestamp = self.ts() + df.delete(timestamp) + # create the baseline hashes file + hashes = df_mgr.get_hashes(self.existing_device, '0', [], policy) + self.assertTrue(suffix in hashes) + # now change the contents of the suffix w/o calling + # invalidate_hash + rmtree(df._datadir) + suffix_path = os.path.dirname(df._datadir) + self.assertTrue(os.path.exists(suffix_path)) # sanity + new_hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + # ... and get_hashes is none the wiser + self.assertEqual(new_hashes, hashes) + + # ... unless remote end asks for a recalc + hashes = df_mgr.get_hashes(self.existing_device, '0', [suffix], + policy) + self.assertNotEqual(new_hashes, hashes) + # and the empty suffix path is removed + self.assertFalse(os.path.exists(suffix_path)) + # ... but is hashed as "empty" + expected = { + EC_POLICY: {}, + REPL_POLICY: md5().hexdigest(), + }[policy.policy_type] + self.assertEqual({suffix: expected}, hashes) + + def test_get_hashes_multi_file_multi_suffix(self): + paths, suffix = find_paths_with_matching_suffixes(needed_matches=2, + needed_suffixes=3) + matching_paths = paths.pop(suffix) + matching_paths.sort(key=lambda path: hash_path(*path)) + other_paths = [] + for suffix, paths in paths.items(): + other_paths.append(paths[0]) + if len(other_paths) >= 2: + break + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # first we'll make a tombstone + df = df_mgr.get_diskfile(self.existing_device, '0', + *other_paths[0], policy=policy, + frag_index=4) + timestamp = self.ts() + df.delete(timestamp) + tombstone_hash = md5(timestamp.internal + '.ts').hexdigest() + tombstone_suffix = os.path.basename(os.path.dirname(df._datadir)) + # second file in another suffix has a .datafile + df = df_mgr.get_diskfile(self.existing_device, '0', + *other_paths[1], policy=policy, + frag_index=5) + timestamp = self.ts() + with df.create() as writer: + test_data = 'test_file' + writer.write(test_data) + metadata = { + 'X-Timestamp': timestamp.internal, + 'ETag': md5(test_data).hexdigest(), + 'Content-Length': len(test_data), + } + writer.put(metadata) + writer.commit(timestamp) + datafile_name = timestamp.internal + if policy.policy_type == EC_POLICY: + datafile_name += '#%d' % df._frag_index + datafile_name += '.data' + durable_hash = md5(timestamp.internal + '.durable').hexdigest() + datafile_suffix = os.path.basename(os.path.dirname(df._datadir)) + # in the *third* suffix - two datafiles for different hashes + df = df_mgr.get_diskfile(self.existing_device, '0', + *matching_paths[0], policy=policy, + frag_index=6) + matching_suffix = os.path.basename(os.path.dirname(df._datadir)) + timestamp = self.ts() + with df.create() as writer: + test_data = 'test_file' + writer.write(test_data) + metadata = { + 'X-Timestamp': timestamp.internal, + 'ETag': md5(test_data).hexdigest(), + 'Content-Length': len(test_data), + } + writer.put(metadata) + writer.commit(timestamp) + # we'll keep track of file names for hash calculations + filename = timestamp.internal + if policy.policy_type == EC_POLICY: + filename += '#%d' % df._frag_index + filename += '.data' + filenames = { + 'data': { + 6: filename + }, + 'durable': [timestamp.internal + '.durable'], + } + df = df_mgr.get_diskfile(self.existing_device, '0', + *matching_paths[1], policy=policy, + frag_index=7) + self.assertEqual(os.path.basename(os.path.dirname(df._datadir)), + matching_suffix) # sanity + timestamp = self.ts() + with df.create() as writer: + test_data = 'test_file' + writer.write(test_data) + metadata = { + 'X-Timestamp': timestamp.internal, + 'ETag': md5(test_data).hexdigest(), + 'Content-Length': len(test_data), + } + writer.put(metadata) + writer.commit(timestamp) + filename = timestamp.internal + if policy.policy_type == EC_POLICY: + filename += '#%d' % df._frag_index + filename += '.data' + filenames['data'][7] = filename + filenames['durable'].append(timestamp.internal + '.durable') + # now make up the expected suffixes! + if policy.policy_type == EC_POLICY: + hasher = md5() + for filename in filenames['durable']: + hasher.update(filename) + expected = { + tombstone_suffix: { + None: tombstone_hash, + }, + datafile_suffix: { + None: durable_hash, + 5: self.fname_to_ts_hash(datafile_name), + }, + matching_suffix: { + None: hasher.hexdigest(), + 6: self.fname_to_ts_hash(filenames['data'][6]), + 7: self.fname_to_ts_hash(filenames['data'][7]), + }, + } + elif policy.policy_type == REPL_POLICY: + hasher = md5() + for filename in filenames['data'].values(): + hasher.update(filename) + expected = { + tombstone_suffix: tombstone_hash, + datafile_suffix: md5(datafile_name).hexdigest(), + matching_suffix: hasher.hexdigest(), + } + else: + self.fail('unknown policy type %r' % policy.policy_type) + hashes = df_mgr.get_hashes('sda1', '0', [], policy) + self.assertEqual(hashes, expected) + + # get_hashes tests - error handling + + def test_get_hashes_bad_dev(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + df_mgr.mount_check = True + with mock.patch('swift.obj.diskfile.check_mount', + mock.MagicMock(side_effect=[False])): + self.assertRaises( + DiskFileDeviceUnavailable, + df_mgr.get_hashes, self.existing_device, '0', ['123'], + policy) + + def test_get_hashes_zero_bytes_pickle(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + part_path = os.path.join(self.devices, self.existing_device, + diskfile.get_data_dir(policy), '0') + os.makedirs(part_path) + # create a pre-existing zero-byte file + open(os.path.join(part_path, diskfile.HASH_FILE), 'w').close() + hashes = df_mgr.get_hashes(self.existing_device, '0', [], + policy) + self.assertEqual(hashes, {}) + + def test_get_hashes_hash_suffix_enotdir(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # create a real suffix dir + df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', + 'o', policy=policy, frag_index=3) + df.delete(Timestamp(time())) + suffix = os.path.basename(os.path.dirname(df._datadir)) + # touch a bad suffix dir + part_dir = os.path.join(self.devices, self.existing_device, + diskfile.get_data_dir(policy), '0') + open(os.path.join(part_dir, 'bad'), 'w').close() + hashes = df_mgr.get_hashes(self.existing_device, '0', [], policy) + self.assertTrue(suffix in hashes) + self.assertFalse('bad' in hashes) + + def test_get_hashes_hash_suffix_other_oserror(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + suffix = '123' + suffix_path = os.path.join(self.devices, self.existing_device, + diskfile.get_data_dir(policy), '0', + suffix) + os.makedirs(suffix_path) + self.assertTrue(os.path.exists(suffix_path)) # sanity + hashes = df_mgr.get_hashes(self.existing_device, '0', [suffix], + policy) + expected = { + EC_POLICY: {'123': {}}, + REPL_POLICY: {'123': EMPTY_ETAG}, + }[policy.policy_type] + msg = 'expected %r != %r for policy %r' % (expected, hashes, + policy) + self.assertEqual(hashes, expected, msg) + + # this OSError does *not* raise PathNotDir, and is allowed to leak + # from hash_suffix into get_hashes + mocked_os_listdir = mock.Mock( + side_effect=OSError(errno.EACCES, os.strerror(errno.EACCES))) + with mock.patch("os.listdir", mocked_os_listdir): + with mock.patch('swift.obj.diskfile.logging') as mock_logging: + hashes = df_mgr.get_hashes('sda1', '0', [suffix], policy) + self.assertEqual(mock_logging.method_calls, + [mock.call.exception('Error hashing suffix')]) + # recalc always causes a suffix to get reset to None; the listdir + # error prevents the suffix from being rehashed + expected = {'123': None} + msg = 'expected %r != %r for policy %r' % (expected, hashes, + policy) + self.assertEqual(hashes, expected, msg) + + def test_get_hashes_modified_recursive_retry(self): + for policy in self.iter_policies(): + df_mgr = self.df_router[policy] + # first create an empty pickle + df_mgr.get_hashes(self.existing_device, '0', [], policy) + hashes_file = os.path.join( + self.devices, self.existing_device, + diskfile.get_data_dir(policy), '0', diskfile.HASH_FILE) + mtime = os.path.getmtime(hashes_file) + non_local = {'mtime': mtime} + + calls = [] + + def mock_getmtime(filename): + t = non_local['mtime'] + if len(calls) <= 3: + # this will make the *next* call get a slightly + # newer mtime than the last + non_local['mtime'] += 1 + # track exactly the value for every return + calls.append(t) + return t + with mock.patch('swift.obj.diskfile.getmtime', + mock_getmtime): + df_mgr.get_hashes(self.existing_device, '0', ['123'], + policy) + + self.assertEqual(calls, [ + mtime + 0, # read + mtime + 1, # modified + mtime + 2, # read + mtime + 3, # modifed + mtime + 4, # read + mtime + 4, # not modifed + ]) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/obj/test_expirer.py b/test/unit/obj/test_expirer.py index 7c174f251..ca815d358 100644 --- a/test/unit/obj/test_expirer.py +++ b/test/unit/obj/test_expirer.py @@ -16,7 +16,7 @@ import urllib from time import time from unittest import main, TestCase -from test.unit import FakeLogger, FakeRing, mocked_http_conn +from test.unit import FakeRing, mocked_http_conn, debug_logger from copy import deepcopy from tempfile import mkdtemp from shutil import rmtree @@ -53,7 +53,8 @@ class TestObjectExpirer(TestCase): internal_client.sleep = not_sleep self.rcache = mkdtemp() - self.logger = FakeLogger() + self.conf = {'recon_cache_path': self.rcache} + self.logger = debug_logger('test-recon') def tearDown(self): rmtree(self.rcache) @@ -167,7 +168,7 @@ class TestObjectExpirer(TestCase): '2': set('5-five 6-six'.split()), '3': set(u'7-seven\u2661'.split()), } - x = ObjectExpirer({}) + x = ObjectExpirer(self.conf) x.swift = InternalClient(containers) deleted_objects = {} @@ -233,31 +234,32 @@ class TestObjectExpirer(TestCase): x = expirer.ObjectExpirer({}, logger=self.logger) x.report() - self.assertEqual(x.logger.log_dict['info'], []) + self.assertEqual(x.logger.get_lines_for_level('info'), []) x.logger._clear() x.report(final=True) - self.assertTrue('completed' in x.logger.log_dict['info'][-1][0][0], - x.logger.log_dict['info']) - self.assertTrue('so far' not in x.logger.log_dict['info'][-1][0][0], - x.logger.log_dict['info']) + self.assertTrue( + 'completed' in str(x.logger.get_lines_for_level('info'))) + self.assertTrue( + 'so far' not in str(x.logger.get_lines_for_level('info'))) x.logger._clear() x.report_last_time = time() - x.report_interval x.report() - self.assertTrue('completed' not in x.logger.log_dict['info'][-1][0][0], - x.logger.log_dict['info']) - self.assertTrue('so far' in x.logger.log_dict['info'][-1][0][0], - x.logger.log_dict['info']) + self.assertTrue( + 'completed' not in str(x.logger.get_lines_for_level('info'))) + self.assertTrue( + 'so far' in str(x.logger.get_lines_for_level('info'))) def test_run_once_nothing_to_do(self): - x = expirer.ObjectExpirer({}, logger=self.logger) + x = expirer.ObjectExpirer(self.conf, logger=self.logger) x.swift = 'throw error because a string does not have needed methods' x.run_once() - self.assertEqual(x.logger.log_dict['exception'], - [(("Unhandled exception",), {}, - "'str' object has no attribute " - "'get_account_info'")]) + self.assertEqual(x.logger.get_lines_for_level('error'), + ["Unhandled exception: "]) + log_args, log_kwargs = x.logger.log_dict['error'][0] + self.assertEqual(str(log_kwargs['exc_info'][1]), + "'str' object has no attribute 'get_account_info'") def test_run_once_calls_report(self): class InternalClient(object): @@ -267,14 +269,14 @@ class TestObjectExpirer(TestCase): def iter_containers(*a, **kw): return [] - x = expirer.ObjectExpirer({}, logger=self.logger) + x = expirer.ObjectExpirer(self.conf, logger=self.logger) x.swift = InternalClient() x.run_once() self.assertEqual( - x.logger.log_dict['info'], - [(('Pass beginning; 1 possible containers; ' - '2 possible objects',), {}), - (('Pass completed in 0s; 0 objects expired',), {})]) + x.logger.get_lines_for_level('info'), [ + 'Pass beginning; 1 possible containers; 2 possible objects', + 'Pass completed in 0s; 0 objects expired', + ]) def test_run_once_unicode_problem(self): class InternalClient(object): @@ -296,7 +298,7 @@ class TestObjectExpirer(TestCase): def delete_container(*a, **kw): pass - x = expirer.ObjectExpirer({}, logger=self.logger) + x = expirer.ObjectExpirer(self.conf, logger=self.logger) x.swift = InternalClient() requests = [] @@ -323,27 +325,28 @@ class TestObjectExpirer(TestCase): def iter_objects(*a, **kw): raise Exception('This should not have been called') - x = expirer.ObjectExpirer({'recon_cache_path': self.rcache}, + x = expirer.ObjectExpirer(self.conf, logger=self.logger) x.swift = InternalClient([{'name': str(int(time() + 86400))}]) x.run_once() - for exccall in x.logger.log_dict['exception']: - self.assertTrue( - 'This should not have been called' not in exccall[0][0]) - self.assertEqual( - x.logger.log_dict['info'], - [(('Pass beginning; 1 possible containers; ' - '2 possible objects',), {}), - (('Pass completed in 0s; 0 objects expired',), {})]) + logs = x.logger.all_log_lines() + self.assertEqual(logs['info'], [ + 'Pass beginning; 1 possible containers; 2 possible objects', + 'Pass completed in 0s; 0 objects expired', + ]) + self.assertTrue('error' not in logs) # Reverse test to be sure it still would blow up the way expected. fake_swift = InternalClient([{'name': str(int(time() - 86400))}]) - x = expirer.ObjectExpirer({}, logger=self.logger, swift=fake_swift) + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) x.run_once() self.assertEqual( - x.logger.log_dict['exception'], - [(('Unhandled exception',), {}, - str(Exception('This should not have been called')))]) + x.logger.get_lines_for_level('error'), [ + 'Unhandled exception: ']) + log_args, log_kwargs = x.logger.log_dict['error'][-1] + self.assertEqual(str(log_kwargs['exc_info'][1]), + 'This should not have been called') def test_object_timestamp_break(self): class InternalClient(object): @@ -369,33 +372,27 @@ class TestObjectExpirer(TestCase): fake_swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % int(time() + 86400)}]) - x = expirer.ObjectExpirer({}, logger=self.logger, swift=fake_swift) + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) x.run_once() - for exccall in x.logger.log_dict['exception']: - self.assertTrue( - 'This should not have been called' not in exccall[0][0]) - self.assertEqual( - x.logger.log_dict['info'], - [(('Pass beginning; 1 possible containers; ' - '2 possible objects',), {}), - (('Pass completed in 0s; 0 objects expired',), {})]) - + self.assertTrue('error' not in x.logger.all_log_lines()) + self.assertEqual(x.logger.get_lines_for_level('info'), [ + 'Pass beginning; 1 possible containers; 2 possible objects', + 'Pass completed in 0s; 0 objects expired', + ]) # Reverse test to be sure it still would blow up the way expected. ts = int(time() - 86400) fake_swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) - x = expirer.ObjectExpirer({}, logger=self.logger, swift=fake_swift) + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) x.delete_actual_object = should_not_be_called x.run_once() - excswhiledeleting = [] - for exccall in x.logger.log_dict['exception']: - if exccall[0][0].startswith('Exception while deleting '): - excswhiledeleting.append(exccall[0][0]) self.assertEqual( - excswhiledeleting, + x.logger.get_lines_for_level('error'), ['Exception while deleting object %d %d-actual-obj ' - 'This should not have been called' % (ts, ts)]) + 'This should not have been called: ' % (ts, ts)]) def test_failed_delete_keeps_entry(self): class InternalClient(object): @@ -428,24 +425,22 @@ class TestObjectExpirer(TestCase): fake_swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) - x = expirer.ObjectExpirer({}, logger=self.logger, swift=fake_swift) + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) x.iter_containers = lambda: [str(int(time() - 86400))] x.delete_actual_object = deliberately_blow_up x.pop_queue = should_not_get_called x.run_once() - excswhiledeleting = [] - for exccall in x.logger.log_dict['exception']: - if exccall[0][0].startswith('Exception while deleting '): - excswhiledeleting.append(exccall[0][0]) + error_lines = x.logger.get_lines_for_level('error') self.assertEqual( - excswhiledeleting, + error_lines, ['Exception while deleting object %d %d-actual-obj ' - 'failed to delete actual object' % (ts, ts)]) + 'failed to delete actual object: ' % (ts, ts)]) self.assertEqual( - x.logger.log_dict['info'], - [(('Pass beginning; 1 possible containers; ' - '2 possible objects',), {}), - (('Pass completed in 0s; 0 objects expired',), {})]) + x.logger.get_lines_for_level('info'), [ + 'Pass beginning; 1 possible containers; 2 possible objects', + 'Pass completed in 0s; 0 objects expired', + ]) # Reverse test to be sure it still would blow up the way expected. ts = int(time() - 86400) @@ -453,18 +448,15 @@ class TestObjectExpirer(TestCase): [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) self.logger._clear() - x = expirer.ObjectExpirer({}, logger=self.logger, swift=fake_swift) + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) x.delete_actual_object = lambda o, t: None x.pop_queue = should_not_get_called x.run_once() - excswhiledeleting = [] - for exccall in x.logger.log_dict['exception']: - if exccall[0][0].startswith('Exception while deleting '): - excswhiledeleting.append(exccall[0][0]) self.assertEqual( - excswhiledeleting, + self.logger.get_lines_for_level('error'), ['Exception while deleting object %d %d-actual-obj This should ' - 'not have been called' % (ts, ts)]) + 'not have been called: ' % (ts, ts)]) def test_success_gets_counted(self): class InternalClient(object): @@ -493,7 +485,8 @@ class TestObjectExpirer(TestCase): fake_swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-acc/c/actual-obj' % int(time() - 86400)}]) - x = expirer.ObjectExpirer({}, logger=self.logger, swift=fake_swift) + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) x.delete_actual_object = lambda o, t: None x.pop_queue = lambda c, o: None self.assertEqual(x.report_objects, 0) @@ -501,10 +494,9 @@ class TestObjectExpirer(TestCase): x.run_once() self.assertEqual(x.report_objects, 1) self.assertEqual( - x.logger.log_dict['info'], - [(('Pass beginning; 1 possible containers; ' - '2 possible objects',), {}), - (('Pass completed in 0s; 1 objects expired',), {})]) + x.logger.get_lines_for_level('info'), + ['Pass beginning; 1 possible containers; 2 possible objects', + 'Pass completed in 0s; 1 objects expired']) def test_delete_actual_object_does_not_get_unicode(self): class InternalClient(object): @@ -539,17 +531,18 @@ class TestObjectExpirer(TestCase): fake_swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': u'%d-actual-obj' % int(time() - 86400)}]) - x = expirer.ObjectExpirer({}, logger=self.logger, swift=fake_swift) + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) x.delete_actual_object = delete_actual_object_test_for_unicode x.pop_queue = lambda c, o: None self.assertEqual(x.report_objects, 0) x.run_once() self.assertEqual(x.report_objects, 1) self.assertEqual( - x.logger.log_dict['info'], - [(('Pass beginning; 1 possible containers; ' - '2 possible objects',), {}), - (('Pass completed in 0s; 1 objects expired',), {})]) + x.logger.get_lines_for_level('info'), [ + 'Pass beginning; 1 possible containers; 2 possible objects', + 'Pass completed in 0s; 1 objects expired', + ]) self.assertFalse(got_unicode[0]) def test_failed_delete_continues_on(self): @@ -579,7 +572,7 @@ class TestObjectExpirer(TestCase): def fail_delete_actual_object(actual_obj, timestamp): raise Exception('failed to delete actual object') - x = expirer.ObjectExpirer({}, logger=self.logger) + x = expirer.ObjectExpirer(self.conf, logger=self.logger) cts = int(time() - 86400) ots = int(time() - 86400) @@ -597,28 +590,24 @@ class TestObjectExpirer(TestCase): x.swift = InternalClient(containers, objects) x.delete_actual_object = fail_delete_actual_object x.run_once() - excswhiledeleting = [] - for exccall in x.logger.log_dict['exception']: - if exccall[0][0].startswith('Exception while deleting '): - excswhiledeleting.append(exccall[0][0]) - self.assertEqual(sorted(excswhiledeleting), sorted([ + error_lines = x.logger.get_lines_for_level('error') + self.assertEqual(sorted(error_lines), sorted([ 'Exception while deleting object %d %d-actual-obj failed to ' - 'delete actual object' % (cts, ots), + 'delete actual object: ' % (cts, ots), 'Exception while deleting object %d %d-next-obj failed to ' - 'delete actual object' % (cts, ots), + 'delete actual object: ' % (cts, ots), 'Exception while deleting object %d %d-actual-obj failed to ' - 'delete actual object' % (cts + 1, ots), + 'delete actual object: ' % (cts + 1, ots), 'Exception while deleting object %d %d-next-obj failed to ' - 'delete actual object' % (cts + 1, ots), + 'delete actual object: ' % (cts + 1, ots), 'Exception while deleting container %d failed to delete ' - 'container' % (cts,), + 'container: ' % (cts,), 'Exception while deleting container %d failed to delete ' - 'container' % (cts + 1,)])) - self.assertEqual( - x.logger.log_dict['info'], - [(('Pass beginning; 1 possible containers; ' - '2 possible objects',), {}), - (('Pass completed in 0s; 0 objects expired',), {})]) + 'container: ' % (cts + 1,)])) + self.assertEqual(x.logger.get_lines_for_level('info'), [ + 'Pass beginning; 1 possible containers; 2 possible objects', + 'Pass completed in 0s; 0 objects expired', + ]) def test_run_forever_initial_sleep_random(self): global last_not_sleep @@ -664,9 +653,11 @@ class TestObjectExpirer(TestCase): finally: expirer.sleep = orig_sleep self.assertEqual(str(err), 'exiting exception 2') - self.assertEqual(x.logger.log_dict['exception'], - [(('Unhandled exception',), {}, - 'exception 1')]) + self.assertEqual(x.logger.get_lines_for_level('error'), + ['Unhandled exception: ']) + log_args, log_kwargs = x.logger.log_dict['error'][0] + self.assertEqual(str(log_kwargs['exc_info'][1]), + 'exception 1') def test_delete_actual_object(self): got_env = [None] diff --git a/test/unit/obj/test_reconstructor.py b/test/unit/obj/test_reconstructor.py new file mode 100755 index 000000000..93a50e84d --- /dev/null +++ b/test/unit/obj/test_reconstructor.py @@ -0,0 +1,2484 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import itertools +import unittest +import os +from hashlib import md5 +import mock +import cPickle as pickle +import tempfile +import time +import shutil +import re +import random +from eventlet import Timeout + +from contextlib import closing, nested, contextmanager +from gzip import GzipFile +from shutil import rmtree +from swift.common import utils +from swift.common.exceptions import DiskFileError +from swift.obj import diskfile, reconstructor as object_reconstructor +from swift.common import ring +from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy, + POLICIES, EC_POLICY) +from swift.obj.reconstructor import REVERT + +from test.unit import (patch_policies, debug_logger, mocked_http_conn, + FabricatedRing, make_timestamp_iter) + + +@contextmanager +def mock_ssync_sender(ssync_calls=None, response_callback=None, **kwargs): + def fake_ssync(daemon, node, job, suffixes): + if ssync_calls is not None: + ssync_calls.append( + {'node': node, 'job': job, 'suffixes': suffixes}) + + def fake_call(): + if response_callback: + response = response_callback(node, job, suffixes) + else: + response = True, {} + return response + return fake_call + + with mock.patch('swift.obj.reconstructor.ssync_sender', fake_ssync): + yield fake_ssync + + +def make_ec_archive_bodies(policy, test_body): + segment_size = policy.ec_segment_size + # split up the body into buffers + chunks = [test_body[x:x + segment_size] + for x in range(0, len(test_body), segment_size)] + # encode the buffers into fragment payloads + fragment_payloads = [] + for chunk in chunks: + fragments = policy.pyeclib_driver.encode(chunk) + if not fragments: + break + fragment_payloads.append(fragments) + + # join up the fragment payloads per node + ec_archive_bodies = [''.join(fragments) + for fragments in zip(*fragment_payloads)] + return ec_archive_bodies + + +def _ips(): + return ['127.0.0.1'] +object_reconstructor.whataremyips = _ips + + +def _create_test_rings(path): + testgz = os.path.join(path, 'object.ring.gz') + intended_replica2part2dev_id = [ + [0, 1, 2], + [1, 2, 3], + [2, 3, 0] + ] + + intended_devs = [ + {'id': 0, 'device': 'sda1', 'zone': 0, 'ip': '127.0.0.0', + 'port': 6000}, + {'id': 1, 'device': 'sda1', 'zone': 1, 'ip': '127.0.0.1', + 'port': 6000}, + {'id': 2, 'device': 'sda1', 'zone': 2, 'ip': '127.0.0.2', + 'port': 6000}, + {'id': 3, 'device': 'sda1', 'zone': 4, 'ip': '127.0.0.3', + 'port': 6000} + ] + intended_part_shift = 30 + with closing(GzipFile(testgz, 'wb')) as f: + pickle.dump( + ring.RingData(intended_replica2part2dev_id, + intended_devs, intended_part_shift), + f) + + testgz = os.path.join(path, 'object-1.ring.gz') + with closing(GzipFile(testgz, 'wb')) as f: + pickle.dump( + ring.RingData(intended_replica2part2dev_id, + intended_devs, intended_part_shift), + f) + + +def count_stats(logger, key, metric): + count = 0 + for record in logger.log_dict[key]: + log_args, log_kwargs = record + m = log_args[0] + if re.match(metric, m): + count += 1 + return count + + +@patch_policies([StoragePolicy(0, name='zero', is_default=True), + ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand', + ec_ndata=2, ec_nparity=1)]) +class TestGlobalSetupObjectReconstructor(unittest.TestCase): + + def setUp(self): + self.testdir = tempfile.mkdtemp() + _create_test_rings(self.testdir) + POLICIES[0].object_ring = ring.Ring(self.testdir, ring_name='object') + POLICIES[1].object_ring = ring.Ring(self.testdir, ring_name='object-1') + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = '' + self.devices = os.path.join(self.testdir, 'node') + os.makedirs(self.devices) + os.mkdir(os.path.join(self.devices, 'sda1')) + self.objects = os.path.join(self.devices, 'sda1', + diskfile.get_data_dir(POLICIES[0])) + self.objects_1 = os.path.join(self.devices, 'sda1', + diskfile.get_data_dir(POLICIES[1])) + os.mkdir(self.objects) + os.mkdir(self.objects_1) + self.parts = {} + self.parts_1 = {} + self.part_nums = ['0', '1', '2'] + for part in self.part_nums: + self.parts[part] = os.path.join(self.objects, part) + os.mkdir(self.parts[part]) + self.parts_1[part] = os.path.join(self.objects_1, part) + os.mkdir(self.parts_1[part]) + + self.conf = dict( + swift_dir=self.testdir, devices=self.devices, mount_check='false', + timeout='300', stats_interval='1') + self.logger = debug_logger('test-reconstructor') + self.reconstructor = object_reconstructor.ObjectReconstructor( + self.conf, logger=self.logger) + + self.policy = POLICIES[1] + + # most of the reconstructor test methods require that there be + # real objects in place, not just part dirs, so we'll create them + # all here.... + # part 0: 3C1/hash/xxx-1.data <-- job: sync_only - parnters (FI 1) + # /xxx.durable <-- included in earlier job (FI 1) + # 061/hash/xxx-1.data <-- included in earlier job (FI 1) + # /xxx.durable <-- included in earlier job (FI 1) + # /xxx-2.data <-- job: sync_revert to index 2 + + # part 1: 3C1/hash/xxx-0.data <-- job: sync_only - parnters (FI 0) + # /xxx-1.data <-- job: sync_revert to index 1 + # /xxx.durable <-- included in earlier jobs (FI 0, 1) + # 061/hash/xxx-1.data <-- included in earlier job (FI 1) + # /xxx.durable <-- included in earlier job (FI 1) + + # part 2: 3C1/hash/xxx-2.data <-- job: sync_revert to index 2 + # /xxx.durable <-- included in earlier job (FI 2) + # 061/hash/xxx-0.data <-- job: sync_revert to index 0 + # /xxx.durable <-- included in earlier job (FI 0) + + def _create_frag_archives(policy, obj_path, local_id, obj_set): + # we'll create 2 sets of objects in different suffix dirs + # so we cover all the scenarios we want (3 of them) + # 1) part dir with all FI's matching the local node index + # 2) part dir with one local and mix of others + # 3) part dir with no local FI and one or more others + def part_0(set): + if set == 0: + # just the local + return local_id + else: + # onde local and all of another + if obj_num == 0: + return local_id + else: + return (local_id + 1) % 3 + + def part_1(set): + if set == 0: + # one local and all of another + if obj_num == 0: + return local_id + else: + return (local_id + 2) % 3 + else: + # just the local node + return local_id + + def part_2(set): + # this part is a handoff in our config (always) + # so lets do a set with indicies from different nodes + if set == 0: + return (local_id + 1) % 3 + else: + return (local_id + 2) % 3 + + # function dictionary for defining test scenarios base on set # + scenarios = {'0': part_0, + '1': part_1, + '2': part_2} + + def _create_df(obj_num, part_num): + self._create_diskfile( + part=part_num, object_name='o' + str(obj_set), + policy=policy, frag_index=scenarios[part_num](obj_set), + timestamp=utils.Timestamp(t)) + + for part_num in self.part_nums: + # create 3 unique objcets per part, each part + # will then have a unique mix of FIs for the + # possible scenarios + for obj_num in range(0, 3): + _create_df(obj_num, part_num) + + ips = utils.whataremyips() + for policy in [p for p in POLICIES if p.policy_type == EC_POLICY]: + self.ec_policy = policy + self.ec_obj_ring = self.reconstructor.load_object_ring( + self.ec_policy) + data_dir = diskfile.get_data_dir(self.ec_policy) + for local_dev in [dev for dev in self.ec_obj_ring.devs + if dev and dev['replication_ip'] in ips and + dev['replication_port'] == + self.reconstructor.port]: + self.ec_local_dev = local_dev + dev_path = os.path.join(self.reconstructor.devices_dir, + self.ec_local_dev['device']) + self.ec_obj_path = os.path.join(dev_path, data_dir) + + # create a bunch of FA's to test + t = 1421181937.70054 # time.time() + with mock.patch('swift.obj.diskfile.time') as mock_time: + # since (a) we are using a fixed time here to create + # frags which corresponds to all the hardcoded hashes and + # (b) the EC diskfile will delete its .data file right + # after creating if it has expired, use this horrible hack + # to prevent the reclaim happening + mock_time.time.return_value = 0.0 + _create_frag_archives(self.ec_policy, self.ec_obj_path, + self.ec_local_dev['id'], 0) + _create_frag_archives(self.ec_policy, self.ec_obj_path, + self.ec_local_dev['id'], 1) + break + break + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def _create_diskfile(self, policy=None, part=0, object_name='o', + frag_index=0, timestamp=None, test_data=None): + policy = policy or self.policy + df_mgr = self.reconstructor._df_router[policy] + df = df_mgr.get_diskfile('sda1', part, 'a', 'c', object_name, + policy=policy) + with df.create() as writer: + timestamp = timestamp or utils.Timestamp(time.time()) + test_data = test_data or 'test data' + writer.write(test_data) + metadata = { + 'X-Timestamp': timestamp.internal, + 'Content-Length': len(test_data), + 'Etag': md5(test_data).hexdigest(), + 'X-Object-Sysmeta-Ec-Frag-Index': frag_index, + } + writer.put(metadata) + writer.commit(timestamp) + return df + + def debug_wtf(self): + # won't include this in the final, just handy reminder of where + # things are... + for pol in [p for p in POLICIES if p.policy_type == EC_POLICY]: + obj_ring = pol.object_ring + for part_num in self.part_nums: + print "\n part_num %s " % part_num + part_nodes = obj_ring.get_part_nodes(int(part_num)) + print "\n part_nodes %s " % part_nodes + for local_dev in obj_ring.devs: + partners = self.reconstructor._get_partners( + local_dev['id'], obj_ring, part_num) + if partners: + print "\n local_dev %s \n partners %s " % (local_dev, + partners) + + def assert_expected_jobs(self, part_num, jobs): + for job in jobs: + del job['path'] + del job['policy'] + if 'local_index' in job: + del job['local_index'] + job['suffixes'].sort() + + expected = [] + # part num 0 + expected.append( + [{ + 'sync_to': [{ + 'index': 2, + 'replication_port': 6000, + 'zone': 2, + 'ip': '127.0.0.2', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.2', + 'device': 'sda1', + 'id': 2, + }], + 'job_type': object_reconstructor.REVERT, + 'suffixes': ['061'], + 'partition': 0, + 'frag_index': 2, + 'device': 'sda1', + 'local_dev': { + 'replication_port': 6000, + 'zone': 1, + 'ip': '127.0.0.1', + 'region': 1, + 'id': 1, + 'replication_ip': '127.0.0.1', + 'device': 'sda1', 'port': 6000, + }, + 'hashes': { + '061': { + None: '85b02a5283704292a511078a5c483da5', + 2: '0e6e8d48d801dc89fd31904ae3b31229', + 1: '0e6e8d48d801dc89fd31904ae3b31229', + }, + '3c1': { + None: '85b02a5283704292a511078a5c483da5', + 1: '0e6e8d48d801dc89fd31904ae3b31229', + }, + }, + }, { + 'sync_to': [{ + 'index': 0, + 'replication_port': 6000, + 'zone': 0, + 'ip': '127.0.0.0', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.0', + 'device': 'sda1', 'id': 0, + }, { + 'index': 2, + 'replication_port': 6000, + 'zone': 2, + 'ip': '127.0.0.2', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.2', + 'device': 'sda1', + 'id': 2, + }], + 'job_type': object_reconstructor.SYNC, + 'sync_diskfile_builder': self.reconstructor.reconstruct_fa, + 'suffixes': ['061', '3c1'], + 'partition': 0, + 'frag_index': 1, + 'device': 'sda1', + 'local_dev': { + 'replication_port': 6000, + 'zone': 1, + 'ip': '127.0.0.1', + 'region': 1, + 'id': 1, + 'replication_ip': '127.0.0.1', + 'device': 'sda1', + 'port': 6000, + }, + 'hashes': + { + '061': { + None: '85b02a5283704292a511078a5c483da5', + 2: '0e6e8d48d801dc89fd31904ae3b31229', + 1: '0e6e8d48d801dc89fd31904ae3b31229' + }, + '3c1': { + None: '85b02a5283704292a511078a5c483da5', + 1: '0e6e8d48d801dc89fd31904ae3b31229', + }, + }, + }] + ) + # part num 1 + expected.append( + [{ + 'sync_to': [{ + 'index': 1, + 'replication_port': 6000, + 'zone': 2, + 'ip': '127.0.0.2', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.2', + 'device': 'sda1', + 'id': 2, + }], + 'job_type': object_reconstructor.REVERT, + 'suffixes': ['061', '3c1'], + 'partition': 1, + 'frag_index': 1, + 'device': 'sda1', + 'local_dev': { + 'replication_port': 6000, + 'zone': 1, + 'ip': '127.0.0.1', + 'region': 1, + 'id': 1, + 'replication_ip': '127.0.0.1', + 'device': 'sda1', + 'port': 6000, + }, + 'hashes': + { + '061': { + None: '85b02a5283704292a511078a5c483da5', + 1: '0e6e8d48d801dc89fd31904ae3b31229', + }, + '3c1': { + 0: '0e6e8d48d801dc89fd31904ae3b31229', + None: '85b02a5283704292a511078a5c483da5', + 1: '0e6e8d48d801dc89fd31904ae3b31229', + }, + }, + }, { + 'sync_to': [{ + 'index': 2, + 'replication_port': 6000, + 'zone': 4, + 'ip': '127.0.0.3', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.3', + 'device': 'sda1', 'id': 3, + }, { + 'index': 1, + 'replication_port': 6000, + 'zone': 2, + 'ip': '127.0.0.2', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.2', + 'device': 'sda1', + 'id': 2, + }], + 'job_type': object_reconstructor.SYNC, + 'sync_diskfile_builder': self.reconstructor.reconstruct_fa, + 'suffixes': ['3c1'], + 'partition': 1, + 'frag_index': 0, + 'device': 'sda1', + 'local_dev': { + 'replication_port': 6000, + 'zone': 1, + 'ip': '127.0.0.1', + 'region': 1, + 'id': 1, + 'replication_ip': '127.0.0.1', + 'device': 'sda1', + 'port': 6000, + }, + 'hashes': { + '061': { + None: '85b02a5283704292a511078a5c483da5', + 1: '0e6e8d48d801dc89fd31904ae3b31229', + }, + '3c1': { + 0: '0e6e8d48d801dc89fd31904ae3b31229', + None: '85b02a5283704292a511078a5c483da5', + 1: '0e6e8d48d801dc89fd31904ae3b31229', + }, + }, + }] + ) + # part num 2 + expected.append( + [{ + 'sync_to': [{ + 'index': 0, + 'replication_port': 6000, + 'zone': 2, + 'ip': '127.0.0.2', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.2', + 'device': 'sda1', 'id': 2, + }], + 'job_type': object_reconstructor.REVERT, + 'suffixes': ['061'], + 'partition': 2, + 'frag_index': 0, + 'device': 'sda1', + 'local_dev': { + 'replication_port': 6000, + 'zone': 1, + 'ip': '127.0.0.1', + 'region': 1, + 'id': 1, + 'replication_ip': '127.0.0.1', + 'device': 'sda1', + 'port': 6000, + }, + 'hashes': { + '061': { + 0: '0e6e8d48d801dc89fd31904ae3b31229', + None: '85b02a5283704292a511078a5c483da5' + }, + '3c1': { + None: '85b02a5283704292a511078a5c483da5', + 2: '0e6e8d48d801dc89fd31904ae3b31229' + }, + }, + }, { + 'sync_to': [{ + 'index': 2, + 'replication_port': 6000, + 'zone': 0, + 'ip': '127.0.0.0', + 'region': 1, + 'port': 6000, + 'replication_ip': '127.0.0.0', + 'device': 'sda1', + 'id': 0, + }], + 'job_type': object_reconstructor.REVERT, + 'suffixes': ['3c1'], + 'partition': 2, + 'frag_index': 2, + 'device': 'sda1', + 'local_dev': { + 'replication_port': 6000, + 'zone': 1, + 'ip': '127.0.0.1', + 'region': 1, + 'id': 1, + 'replication_ip': '127.0.0.1', + 'device': 'sda1', + 'port': 6000 + }, + 'hashes': { + '061': { + 0: '0e6e8d48d801dc89fd31904ae3b31229', + None: '85b02a5283704292a511078a5c483da5' + }, + '3c1': { + None: '85b02a5283704292a511078a5c483da5', + 2: '0e6e8d48d801dc89fd31904ae3b31229' + }, + }, + }] + ) + + def check_jobs(part_num): + try: + expected_jobs = expected[int(part_num)] + except (IndexError, ValueError): + self.fail('Unknown part number %r' % part_num) + expected_by_part_frag_index = dict( + ((j['partition'], j['frag_index']), j) for j in expected_jobs) + for job in jobs: + job_key = (job['partition'], job['frag_index']) + if job_key in expected_by_part_frag_index: + for k, value in job.items(): + expected_value = \ + expected_by_part_frag_index[job_key][k] + try: + if isinstance(value, list): + value.sort() + expected_value.sort() + self.assertEqual(value, expected_value) + except AssertionError as e: + extra_info = \ + '\n\n... for %r in part num %s job %r' % ( + k, part_num, job_key) + raise AssertionError(str(e) + extra_info) + else: + self.fail( + 'Unexpected job %r for part num %s - ' + 'expected jobs where %r' % ( + job_key, part_num, + expected_by_part_frag_index.keys())) + for expected_job in expected_jobs: + if expected_job in jobs: + jobs.remove(expected_job) + self.assertFalse(jobs) # that should be all of them + check_jobs(part_num) + + def test_run_once(self): + with mocked_http_conn(*[200] * 12, body=pickle.dumps({})): + with mock_ssync_sender(): + self.reconstructor.run_once() + + def test_get_response(self): + part = self.part_nums[0] + node = POLICIES[0].object_ring.get_part_nodes(int(part))[0] + for stat_code in (200, 400): + with mocked_http_conn(stat_code): + resp = self.reconstructor._get_response(node, part, + path='nada', + headers={}, + policy=POLICIES[0]) + if resp: + self.assertEqual(resp.status, 200) + else: + self.assertEqual( + len(self.reconstructor.logger.log_dict['warning']), 1) + + def test_reconstructor_skips_bogus_partition_dirs(self): + # A directory in the wrong place shouldn't crash the reconstructor + rmtree(self.objects_1) + os.mkdir(self.objects_1) + + os.mkdir(os.path.join(self.objects_1, "burrito")) + jobs = [] + for part_info in self.reconstructor.collect_parts(): + jobs += self.reconstructor.build_reconstruction_jobs(part_info) + self.assertEqual(len(jobs), 0) + + def test_check_ring(self): + testring = tempfile.mkdtemp() + _create_test_rings(testring) + obj_ring = ring.Ring(testring, ring_name='object') # noqa + self.assertTrue(self.reconstructor.check_ring(obj_ring)) + orig_check = self.reconstructor.next_check + self.reconstructor.next_check = orig_check - 30 + self.assertTrue(self.reconstructor.check_ring(obj_ring)) + self.reconstructor.next_check = orig_check + orig_ring_time = obj_ring._mtime + obj_ring._mtime = orig_ring_time - 30 + self.assertTrue(self.reconstructor.check_ring(obj_ring)) + self.reconstructor.next_check = orig_check - 30 + self.assertFalse(self.reconstructor.check_ring(obj_ring)) + rmtree(testring, ignore_errors=1) + + def test_build_reconstruction_jobs(self): + self.reconstructor.handoffs_first = False + self.reconstructor._reset_stats() + for part_info in self.reconstructor.collect_parts(): + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertTrue(jobs[0]['job_type'] in + (object_reconstructor.SYNC, + object_reconstructor.REVERT)) + self.assert_expected_jobs(part_info['partition'], jobs) + + self.reconstructor.handoffs_first = True + self.reconstructor._reset_stats() + for part_info in self.reconstructor.collect_parts(): + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertTrue(jobs[0]['job_type'] == + object_reconstructor.REVERT) + self.assert_expected_jobs(part_info['partition'], jobs) + + def test_get_partners(self): + # we're going to perform an exhaustive test of every possible + # combination of partitions and nodes in our custom test ring + + # format: [dev_id in question, 'part_num', + # [part_nodes for the given part], left id, right id...] + expected_partners = sorted([ + (0, '0', [0, 1, 2], 2, 1), (0, '2', [2, 3, 0], 3, 2), + (1, '0', [0, 1, 2], 0, 2), (1, '1', [1, 2, 3], 3, 2), + (2, '0', [0, 1, 2], 1, 0), (2, '1', [1, 2, 3], 1, 3), + (2, '2', [2, 3, 0], 0, 3), (3, '1', [1, 2, 3], 2, 1), + (3, '2', [2, 3, 0], 2, 0), (0, '0', [0, 1, 2], 2, 1), + (0, '2', [2, 3, 0], 3, 2), (1, '0', [0, 1, 2], 0, 2), + (1, '1', [1, 2, 3], 3, 2), (2, '0', [0, 1, 2], 1, 0), + (2, '1', [1, 2, 3], 1, 3), (2, '2', [2, 3, 0], 0, 3), + (3, '1', [1, 2, 3], 2, 1), (3, '2', [2, 3, 0], 2, 0), + ]) + + got_partners = [] + for pol in POLICIES: + obj_ring = pol.object_ring + for part_num in self.part_nums: + part_nodes = obj_ring.get_part_nodes(int(part_num)) + primary_ids = [n['id'] for n in part_nodes] + for node in part_nodes: + partners = self.reconstructor._get_partners( + node['index'], part_nodes) + left = partners[0]['id'] + right = partners[1]['id'] + got_partners.append(( + node['id'], part_num, primary_ids, left, right)) + + self.assertEqual(expected_partners, sorted(got_partners)) + + def test_collect_parts(self): + parts = [] + for part_info in self.reconstructor.collect_parts(): + parts.append(part_info['partition']) + self.assertEqual(sorted(parts), [0, 1, 2]) + + def test_collect_parts_mkdirs_error(self): + + def blowup_mkdirs(path): + raise OSError('Ow!') + + with mock.patch.object(object_reconstructor, 'mkdirs', blowup_mkdirs): + rmtree(self.objects_1, ignore_errors=1) + parts = [] + for part_info in self.reconstructor.collect_parts(): + parts.append(part_info['partition']) + error_lines = self.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + log_args, log_kwargs = self.logger.log_dict['error'][0] + self.assertEquals(str(log_kwargs['exc_info'][1]), 'Ow!') + + def test_removes_zbf(self): + # After running xfs_repair, a partition directory could become a + # zero-byte file. If this happens, the reconstructor should clean it + # up, log something, and move on to the next partition. + + # Surprise! Partition dir 1 is actually a zero-byte file. + pol_1_part_1_path = os.path.join(self.objects_1, '1') + rmtree(pol_1_part_1_path) + with open(pol_1_part_1_path, 'w'): + pass + self.assertTrue(os.path.isfile(pol_1_part_1_path)) # sanity check + + # since our collect_parts job is a generator, that yields directly + # into build_jobs and then spawns it's safe to do the remove_files + # without making reconstructor startup slow + for part_info in self.reconstructor.collect_parts(): + self.assertNotEqual(pol_1_part_1_path, part_info['part_path']) + self.assertFalse(os.path.exists(pol_1_part_1_path)) + warnings = self.reconstructor.logger.get_lines_for_level('warning') + self.assertEqual(1, len(warnings)) + self.assertTrue('Unexpected entity in data dir:' in warnings[0], + 'Warning not found in %s' % warnings) + + def _make_fake_ssync(self, ssync_calls): + class _fake_ssync(object): + def __init__(self, daemon, node, job, suffixes, **kwargs): + # capture context and generate an available_map of objs + context = {} + context['node'] = node + context['job'] = job + context['suffixes'] = suffixes + self.suffixes = suffixes + self.daemon = daemon + self.job = job + hash_gen = self.daemon._diskfile_mgr.yield_hashes( + self.job['device'], self.job['partition'], + self.job['policy'], self.suffixes, + frag_index=self.job.get('frag_index')) + self.available_map = {} + for path, hash_, ts in hash_gen: + self.available_map[hash_] = ts + context['available_map'] = self.available_map + ssync_calls.append(context) + + def __call__(self, *args, **kwargs): + return True, self.available_map + + return _fake_ssync + + def test_delete_reverted(self): + # verify reconstructor deletes reverted frag indexes after ssync'ing + + def visit_obj_dirs(context): + for suff in context['suffixes']: + suff_dir = os.path.join( + context['job']['path'], suff) + for root, dirs, files in os.walk(suff_dir): + for d in dirs: + dirpath = os.path.join(root, d) + files = os.listdir(dirpath) + yield dirpath, files + + n_files = n_files_after = 0 + + # run reconstructor with delete function mocked out to check calls + ssync_calls = [] + delete_func =\ + 'swift.obj.reconstructor.ObjectReconstructor.delete_reverted_objs' + with mock.patch('swift.obj.reconstructor.ssync_sender', + self._make_fake_ssync(ssync_calls)): + with mocked_http_conn(*[200] * 12, body=pickle.dumps({})): + with mock.patch(delete_func) as mock_delete: + self.reconstructor.reconstruct() + expected_calls = [] + for context in ssync_calls: + if context['job']['job_type'] == REVERT: + for dirpath, files in visit_obj_dirs(context): + # sanity check - expect some files to be in dir, + # may not be for the reverted frag index + self.assertTrue(files) + n_files += len(files) + expected_calls.append(mock.call(context['job'], + context['available_map'], + context['node']['index'])) + mock_delete.assert_has_calls(expected_calls, any_order=True) + + ssync_calls = [] + with mock.patch('swift.obj.reconstructor.ssync_sender', + self._make_fake_ssync(ssync_calls)): + with mocked_http_conn(*[200] * 12, body=pickle.dumps({})): + self.reconstructor.reconstruct() + for context in ssync_calls: + if context['job']['job_type'] == REVERT: + data_file_tail = ('#%s.data' + % context['node']['index']) + for dirpath, files in visit_obj_dirs(context): + n_files_after += len(files) + for filename in files: + self.assertFalse( + filename.endswith(data_file_tail)) + + # sanity check that some files should were deleted + self.assertTrue(n_files > n_files_after) + + def test_get_part_jobs(self): + # yeah, this test code expects a specific setup + self.assertEqual(len(self.part_nums), 3) + + # OK, at this point we should have 4 loaded parts with one + jobs = [] + for partition in os.listdir(self.ec_obj_path): + part_path = os.path.join(self.ec_obj_path, partition) + jobs = self.reconstructor._get_part_jobs( + self.ec_local_dev, part_path, int(partition), self.ec_policy) + self.assert_expected_jobs(partition, jobs) + + def assertStatCount(self, stat_method, stat_prefix, expected_count): + count = count_stats(self.logger, stat_method, stat_prefix) + msg = 'expected %s != %s for %s %s' % ( + expected_count, count, stat_method, stat_prefix) + self.assertEqual(expected_count, count, msg) + + def test_delete_partition(self): + # part 2 is predefined to have all revert jobs + part_path = os.path.join(self.objects_1, '2') + self.assertTrue(os.access(part_path, os.F_OK)) + + ssync_calls = [] + status = [200] * 2 + body = pickle.dumps({}) + with mocked_http_conn(*status, body=body) as request_log: + with mock.patch('swift.obj.reconstructor.ssync_sender', + self._make_fake_ssync(ssync_calls)): + self.reconstructor.reconstruct(override_partitions=[2]) + expected_repliate_calls = set([ + ('127.0.0.0', '/sda1/2/3c1'), + ('127.0.0.2', '/sda1/2/061'), + ]) + found_calls = set((r['ip'], r['path']) + for r in request_log.requests) + self.assertEqual(expected_repliate_calls, found_calls) + + expected_ssync_calls = sorted([ + ('127.0.0.0', REVERT, 2, ['3c1']), + ('127.0.0.2', REVERT, 2, ['061']), + ]) + self.assertEqual(expected_ssync_calls, sorted(( + c['node']['ip'], + c['job']['job_type'], + c['job']['partition'], + c['suffixes'], + ) for c in ssync_calls)) + + expected_stats = { + ('increment', 'partition.delete.count.'): 2, + ('timing_since', 'partition.delete.timing'): 2, + } + for stat_key, expected in expected_stats.items(): + stat_method, stat_prefix = stat_key + self.assertStatCount(stat_method, stat_prefix, expected) + # part 2 should be totally empty + policy = POLICIES[1] + hash_gen = self.reconstructor._df_router[policy].yield_hashes( + 'sda1', '2', policy) + for path, hash_, ts in hash_gen: + self.fail('found %s with %s in %s', (hash_, ts, path)) + # but the partition directory and hashes pkl still exist + self.assertTrue(os.access(part_path, os.F_OK)) + hashes_path = os.path.join(self.objects_1, '2', diskfile.HASH_FILE) + self.assertTrue(os.access(hashes_path, os.F_OK)) + + # ... but on next pass + ssync_calls = [] + with mocked_http_conn() as request_log: + with mock.patch('swift.obj.reconstructor.ssync_sender', + self._make_fake_ssync(ssync_calls)): + self.reconstructor.reconstruct(override_partitions=[2]) + # reconstruct won't generate any replicate or ssync_calls + self.assertFalse(request_log.requests) + self.assertFalse(ssync_calls) + # and the partition will get removed! + self.assertFalse(os.access(part_path, os.F_OK)) + + def test_process_job_all_success(self): + self.reconstructor._reset_stats() + with mock_ssync_sender(): + with mocked_http_conn(*[200] * 12, body=pickle.dumps({})): + found_jobs = [] + for part_info in self.reconstructor.collect_parts(): + jobs = self.reconstructor.build_reconstruction_jobs( + part_info) + found_jobs.extend(jobs) + for job in jobs: + self.logger._clear() + node_count = len(job['sync_to']) + self.reconstructor.process_job(job) + if job['job_type'] == object_reconstructor.REVERT: + self.assertEqual(0, count_stats( + self.logger, 'update_stats', 'suffix.hashes')) + else: + self.assertStatCount('update_stats', + 'suffix.hashes', + node_count) + self.assertEqual(node_count, count_stats( + self.logger, 'update_stats', 'suffix.hashes')) + self.assertEqual(node_count, count_stats( + self.logger, 'update_stats', 'suffix.syncs')) + self.assertFalse('error' in + self.logger.all_log_lines()) + self.assertEqual(self.reconstructor.suffix_sync, 8) + self.assertEqual(self.reconstructor.suffix_count, 8) + self.assertEqual(len(found_jobs), 6) + + def test_process_job_all_insufficient_storage(self): + self.reconstructor._reset_stats() + with mock_ssync_sender(): + with mocked_http_conn(*[507] * 10): + found_jobs = [] + for part_info in self.reconstructor.collect_parts(): + jobs = self.reconstructor.build_reconstruction_jobs( + part_info) + found_jobs.extend(jobs) + for job in jobs: + self.logger._clear() + self.reconstructor.process_job(job) + for line in self.logger.get_lines_for_level('error'): + self.assertTrue('responded as unmounted' in line) + self.assertEqual(0, count_stats( + self.logger, 'update_stats', 'suffix.hashes')) + self.assertEqual(0, count_stats( + self.logger, 'update_stats', 'suffix.syncs')) + self.assertEqual(self.reconstructor.suffix_sync, 0) + self.assertEqual(self.reconstructor.suffix_count, 0) + self.assertEqual(len(found_jobs), 6) + + def test_process_job_all_client_error(self): + self.reconstructor._reset_stats() + with mock_ssync_sender(): + with mocked_http_conn(*[400] * 10): + found_jobs = [] + for part_info in self.reconstructor.collect_parts(): + jobs = self.reconstructor.build_reconstruction_jobs( + part_info) + found_jobs.extend(jobs) + for job in jobs: + self.logger._clear() + self.reconstructor.process_job(job) + for line in self.logger.get_lines_for_level('error'): + self.assertTrue('Invalid response 400' in line) + self.assertEqual(0, count_stats( + self.logger, 'update_stats', 'suffix.hashes')) + self.assertEqual(0, count_stats( + self.logger, 'update_stats', 'suffix.syncs')) + self.assertEqual(self.reconstructor.suffix_sync, 0) + self.assertEqual(self.reconstructor.suffix_count, 0) + self.assertEqual(len(found_jobs), 6) + + def test_process_job_all_timeout(self): + self.reconstructor._reset_stats() + with mock_ssync_sender(): + with nested(mocked_http_conn(*[Timeout()] * 10)): + found_jobs = [] + for part_info in self.reconstructor.collect_parts(): + jobs = self.reconstructor.build_reconstruction_jobs( + part_info) + found_jobs.extend(jobs) + for job in jobs: + self.logger._clear() + self.reconstructor.process_job(job) + for line in self.logger.get_lines_for_level('error'): + self.assertTrue('Timeout (Nones)' in line) + self.assertStatCount( + 'update_stats', 'suffix.hashes', 0) + self.assertStatCount( + 'update_stats', 'suffix.syncs', 0) + self.assertEqual(self.reconstructor.suffix_sync, 0) + self.assertEqual(self.reconstructor.suffix_count, 0) + self.assertEqual(len(found_jobs), 6) + + +@patch_policies(with_ec_default=True) +class TestObjectReconstructor(unittest.TestCase): + + def setUp(self): + self.policy = POLICIES.default + self.testdir = tempfile.mkdtemp() + self.devices = os.path.join(self.testdir, 'devices') + self.local_dev = self.policy.object_ring.devs[0] + self.ip = self.local_dev['replication_ip'] + self.port = self.local_dev['replication_port'] + self.conf = { + 'devices': self.devices, + 'mount_check': False, + 'bind_port': self.port, + } + self.logger = debug_logger('object-reconstructor') + self.reconstructor = object_reconstructor.ObjectReconstructor( + self.conf, logger=self.logger) + self.reconstructor._reset_stats() + # some tests bypass build_reconstruction_jobs and go to process_job + # directly, so you end up with a /0 when you try to show the + # percentage of complete jobs as ratio of the total job count + self.reconstructor.job_count = 1 + self.policy.object_ring.max_more_nodes = \ + self.policy.object_ring.replicas + self.ts_iter = make_timestamp_iter() + + def tearDown(self): + self.reconstructor.stats_line() + shutil.rmtree(self.testdir) + + def ts(self): + return next(self.ts_iter) + + def test_collect_parts_skips_non_ec_policy_and_device(self): + stub_parts = (371, 78, 419, 834) + for policy in POLICIES: + datadir = diskfile.get_data_dir(policy) + for part in stub_parts: + utils.mkdirs(os.path.join( + self.devices, self.local_dev['device'], + datadir, str(part))) + with mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]): + part_infos = list(self.reconstructor.collect_parts()) + found_parts = sorted(int(p['partition']) for p in part_infos) + self.assertEqual(found_parts, sorted(stub_parts)) + for part_info in part_infos: + self.assertEqual(part_info['local_dev'], self.local_dev) + self.assertEqual(part_info['policy'], self.policy) + self.assertEqual(part_info['part_path'], + os.path.join(self.devices, + self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(part_info['partition']))) + + def test_collect_parts_multi_device_skips_non_ring_devices(self): + device_parts = { + 'sda': (374,), + 'sdb': (179, 807), + 'sdc': (363, 468, 843), + } + for policy in POLICIES: + datadir = diskfile.get_data_dir(policy) + for dev, parts in device_parts.items(): + for part in parts: + utils.mkdirs(os.path.join( + self.devices, dev, + datadir, str(part))) + + # we're only going to add sda and sdc into the ring + local_devs = ('sda', 'sdc') + stub_ring_devs = [{ + 'device': dev, + 'replication_ip': self.ip, + 'replication_port': self.port + } for dev in local_devs] + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch.object(self.policy.object_ring, '_devs', + new=stub_ring_devs)): + part_infos = list(self.reconstructor.collect_parts()) + found_parts = sorted(int(p['partition']) for p in part_infos) + expected_parts = sorted(itertools.chain( + *(device_parts[d] for d in local_devs))) + self.assertEqual(found_parts, expected_parts) + for part_info in part_infos: + self.assertEqual(part_info['policy'], self.policy) + self.assertTrue(part_info['local_dev'] in stub_ring_devs) + dev = part_info['local_dev'] + self.assertEqual(part_info['part_path'], + os.path.join(self.devices, + dev['device'], + diskfile.get_data_dir(self.policy), + str(part_info['partition']))) + + def test_collect_parts_mount_check(self): + # each device has one part in it + local_devs = ('sda', 'sdb') + for i, dev in enumerate(local_devs): + datadir = diskfile.get_data_dir(self.policy) + utils.mkdirs(os.path.join( + self.devices, dev, datadir, str(i))) + stub_ring_devs = [{ + 'device': dev, + 'replication_ip': self.ip, + 'replication_port': self.port + } for dev in local_devs] + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch.object(self.policy.object_ring, '_devs', + new=stub_ring_devs)): + part_infos = list(self.reconstructor.collect_parts()) + self.assertEqual(2, len(part_infos)) # sanity + self.assertEqual(set(int(p['partition']) for p in part_infos), + set([0, 1])) + + paths = [] + + def fake_ismount(path): + paths.append(path) + return False + + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch.object(self.policy.object_ring, '_devs', + new=stub_ring_devs), + mock.patch('swift.obj.reconstructor.ismount', + fake_ismount)): + part_infos = list(self.reconstructor.collect_parts()) + self.assertEqual(2, len(part_infos)) # sanity, same jobs + self.assertEqual(set(int(p['partition']) for p in part_infos), + set([0, 1])) + + # ... because ismount was not called + self.assertEqual(paths, []) + + # ... now with mount check + self.reconstructor.mount_check = True + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch.object(self.policy.object_ring, '_devs', + new=stub_ring_devs), + mock.patch('swift.obj.reconstructor.ismount', + fake_ismount)): + part_infos = list(self.reconstructor.collect_parts()) + self.assertEqual([], part_infos) # sanity, no jobs + + # ... because fake_ismount returned False for both paths + self.assertEqual(set(paths), set([ + os.path.join(self.devices, dev) for dev in local_devs])) + + def fake_ismount(path): + if path.endswith('sda'): + return True + else: + return False + + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch.object(self.policy.object_ring, '_devs', + new=stub_ring_devs), + mock.patch('swift.obj.reconstructor.ismount', + fake_ismount)): + part_infos = list(self.reconstructor.collect_parts()) + self.assertEqual(1, len(part_infos)) # only sda picked up (part 0) + self.assertEqual(part_infos[0]['partition'], 0) + + def test_collect_parts_cleans_tmp(self): + local_devs = ('sda', 'sdc') + stub_ring_devs = [{ + 'device': dev, + 'replication_ip': self.ip, + 'replication_port': self.port + } for dev in local_devs] + fake_unlink = mock.MagicMock() + self.reconstructor.reclaim_age = 1000 + now = time.time() + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch('swift.obj.reconstructor.time.time', + return_value=now), + mock.patch.object(self.policy.object_ring, '_devs', + new=stub_ring_devs), + mock.patch('swift.obj.reconstructor.unlink_older_than', + fake_unlink)): + self.assertEqual([], list(self.reconstructor.collect_parts())) + # each local device hash unlink_older_than called on it, + # with now - self.reclaim_age + tmpdir = diskfile.get_tmp_dir(self.policy) + expected = now - 1000 + self.assertEqual(fake_unlink.mock_calls, [ + mock.call(os.path.join(self.devices, dev, tmpdir), expected) + for dev in local_devs]) + + def test_collect_parts_creates_datadir(self): + # create just the device path + dev_path = os.path.join(self.devices, self.local_dev['device']) + utils.mkdirs(dev_path) + with mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]): + self.assertEqual([], list(self.reconstructor.collect_parts())) + datadir_path = os.path.join(dev_path, + diskfile.get_data_dir(self.policy)) + self.assertTrue(os.path.exists(datadir_path)) + + def test_collect_parts_creates_datadir_error(self): + # create just the device path + datadir_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy)) + utils.mkdirs(os.path.dirname(datadir_path)) + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch('swift.obj.reconstructor.mkdirs', + side_effect=OSError('kaboom!'))): + self.assertEqual([], list(self.reconstructor.collect_parts())) + error_lines = self.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + line = error_lines[0] + self.assertTrue('Unable to create' in line) + self.assertTrue(datadir_path in line) + + def test_collect_parts_skips_invalid_paths(self): + datadir_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy)) + utils.mkdirs(os.path.dirname(datadir_path)) + with open(datadir_path, 'w') as f: + f.write('junk') + with mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]): + self.assertEqual([], list(self.reconstructor.collect_parts())) + self.assertTrue(os.path.exists(datadir_path)) + error_lines = self.logger.get_lines_for_level('error') + self.assertEqual(len(error_lines), 1) + line = error_lines[0] + self.assertTrue('Unable to list partitions' in line) + self.assertTrue(datadir_path in line) + + def test_collect_parts_removes_non_partition_files(self): + # create some junk next to partitions + datadir_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy)) + num_parts = 3 + for part in range(num_parts): + utils.mkdirs(os.path.join(datadir_path, str(part))) + junk_file = os.path.join(datadir_path, 'junk') + with open(junk_file, 'w') as f: + f.write('junk') + with mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]): + part_infos = list(self.reconstructor.collect_parts()) + # the file is not included in the part_infos map + self.assertEqual(sorted(p['part_path'] for p in part_infos), + sorted([os.path.join(datadir_path, str(i)) + for i in range(num_parts)])) + # and gets cleaned up + self.assertFalse(os.path.exists(junk_file)) + + def test_collect_parts_overrides(self): + # setup multiple devices, with multiple parts + device_parts = { + 'sda': (374, 843), + 'sdb': (179, 807), + 'sdc': (363, 468, 843), + } + datadir = diskfile.get_data_dir(self.policy) + for dev, parts in device_parts.items(): + for part in parts: + utils.mkdirs(os.path.join( + self.devices, dev, + datadir, str(part))) + + # we're only going to add sda and sdc into the ring + local_devs = ('sda', 'sdc') + stub_ring_devs = [{ + 'device': dev, + 'replication_ip': self.ip, + 'replication_port': self.port + } for dev in local_devs] + + expected = ( + ({}, [ + ('sda', 374), + ('sda', 843), + ('sdc', 363), + ('sdc', 468), + ('sdc', 843), + ]), + ({'override_devices': ['sda', 'sdc']}, [ + ('sda', 374), + ('sda', 843), + ('sdc', 363), + ('sdc', 468), + ('sdc', 843), + ]), + ({'override_devices': ['sdc']}, [ + ('sdc', 363), + ('sdc', 468), + ('sdc', 843), + ]), + ({'override_devices': ['sda']}, [ + ('sda', 374), + ('sda', 843), + ]), + ({'override_devices': ['sdx']}, []), + ({'override_partitions': [374]}, [ + ('sda', 374), + ]), + ({'override_partitions': [843]}, [ + ('sda', 843), + ('sdc', 843), + ]), + ({'override_partitions': [843], 'override_devices': ['sda']}, [ + ('sda', 843), + ]), + ) + with nested(mock.patch('swift.obj.reconstructor.whataremyips', + return_value=[self.ip]), + mock.patch.object(self.policy.object_ring, '_devs', + new=stub_ring_devs)): + for kwargs, expected_parts in expected: + part_infos = list(self.reconstructor.collect_parts(**kwargs)) + expected_paths = set( + os.path.join(self.devices, dev, datadir, str(part)) + for dev, part in expected_parts) + found_paths = set(p['part_path'] for p in part_infos) + msg = 'expected %r != %r for %r' % ( + expected_paths, found_paths, kwargs) + self.assertEqual(expected_paths, found_paths, msg) + + def test_build_jobs_creates_empty_hashes(self): + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), '0') + utils.mkdirs(part_path) + part_info = { + 'local_dev': self.local_dev, + 'policy': self.policy, + 'partition': 0, + 'part_path': part_path, + } + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertEqual(1, len(jobs)) + job = jobs[0] + self.assertEqual(job['job_type'], object_reconstructor.SYNC) + self.assertEqual(job['frag_index'], 0) + self.assertEqual(job['suffixes'], []) + self.assertEqual(len(job['sync_to']), 2) + self.assertEqual(job['partition'], 0) + self.assertEqual(job['path'], part_path) + self.assertEqual(job['hashes'], {}) + self.assertEqual(job['policy'], self.policy) + self.assertEqual(job['local_dev'], self.local_dev) + self.assertEqual(job['device'], self.local_dev['device']) + hashes_file = os.path.join(part_path, + diskfile.HASH_FILE) + self.assertTrue(os.path.exists(hashes_file)) + suffixes = self.reconstructor._get_hashes( + self.policy, part_path, do_listdir=True) + self.assertEqual(suffixes, {}) + + def test_build_jobs_no_hashes(self): + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), '0') + part_info = { + 'local_dev': self.local_dev, + 'policy': self.policy, + 'partition': 0, + 'part_path': part_path, + } + stub_hashes = {} + with mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes)): + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertEqual(1, len(jobs)) + job = jobs[0] + self.assertEqual(job['job_type'], object_reconstructor.SYNC) + self.assertEqual(job['frag_index'], 0) + self.assertEqual(job['suffixes'], []) + self.assertEqual(len(job['sync_to']), 2) + self.assertEqual(job['partition'], 0) + self.assertEqual(job['path'], part_path) + self.assertEqual(job['hashes'], {}) + self.assertEqual(job['policy'], self.policy) + self.assertEqual(job['local_dev'], self.local_dev) + self.assertEqual(job['device'], self.local_dev['device']) + + def test_build_jobs_primary(self): + ring = self.policy.object_ring = FabricatedRing() + # find a partition for which we're a primary + for partition in range(2 ** ring.part_power): + part_nodes = ring.get_part_nodes(partition) + try: + frag_index = [n['id'] for n in part_nodes].index( + self.local_dev['id']) + except ValueError: + pass + else: + break + else: + self.fail("the ring doesn't work: %r" % ring._replica2part2dev_id) + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + part_info = { + 'local_dev': self.local_dev, + 'policy': self.policy, + 'partition': partition, + 'part_path': part_path, + } + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + with mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes)): + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertEqual(1, len(jobs)) + job = jobs[0] + self.assertEqual(job['job_type'], object_reconstructor.SYNC) + self.assertEqual(job['frag_index'], frag_index) + self.assertEqual(job['suffixes'], stub_hashes.keys()) + self.assertEqual(set([n['index'] for n in job['sync_to']]), + set([(frag_index + 1) % ring.replicas, + (frag_index - 1) % ring.replicas])) + self.assertEqual(job['partition'], partition) + self.assertEqual(job['path'], part_path) + self.assertEqual(job['hashes'], stub_hashes) + self.assertEqual(job['policy'], self.policy) + self.assertEqual(job['local_dev'], self.local_dev) + self.assertEqual(job['device'], self.local_dev['device']) + + def test_build_jobs_handoff(self): + ring = self.policy.object_ring = FabricatedRing() + # find a partition for which we're a handoff + for partition in range(2 ** ring.part_power): + part_nodes = ring.get_part_nodes(partition) + if self.local_dev['id'] not in [n['id'] for n in part_nodes]: + break + else: + self.fail("the ring doesn't work: %r" % ring._replica2part2dev_id) + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + part_info = { + 'local_dev': self.local_dev, + 'policy': self.policy, + 'partition': partition, + 'part_path': part_path, + } + # since this part doesn't belong on us it doesn't matter what + # frag_index we have + frag_index = random.randint(0, ring.replicas - 1) + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {None: 'hash'}, + } + with mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes)): + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertEqual(1, len(jobs)) + job = jobs[0] + self.assertEqual(job['job_type'], object_reconstructor.REVERT) + self.assertEqual(job['frag_index'], frag_index) + self.assertEqual(sorted(job['suffixes']), sorted(stub_hashes.keys())) + self.assertEqual(len(job['sync_to']), 1) + self.assertEqual(job['sync_to'][0]['index'], frag_index) + self.assertEqual(job['path'], part_path) + self.assertEqual(job['partition'], partition) + self.assertEqual(sorted(job['hashes']), sorted(stub_hashes)) + self.assertEqual(job['local_dev'], self.local_dev) + + def test_build_jobs_mixed(self): + ring = self.policy.object_ring = FabricatedRing() + # find a partition for which we're a primary + for partition in range(2 ** ring.part_power): + part_nodes = ring.get_part_nodes(partition) + try: + frag_index = [n['id'] for n in part_nodes].index( + self.local_dev['id']) + except ValueError: + pass + else: + break + else: + self.fail("the ring doesn't work: %r" % ring._replica2part2dev_id) + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + part_info = { + 'local_dev': self.local_dev, + 'policy': self.policy, + 'partition': partition, + 'part_path': part_path, + } + other_frag_index = random.choice([f for f in range(ring.replicas) + if f != frag_index]) + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + '456': {other_frag_index: 'hash', None: 'hash'}, + 'abc': {None: 'hash'}, + } + with mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes)): + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertEqual(2, len(jobs)) + sync_jobs, revert_jobs = [], [] + for job in jobs: + self.assertEqual(job['partition'], partition) + self.assertEqual(job['path'], part_path) + self.assertEqual(sorted(job['hashes']), sorted(stub_hashes)) + self.assertEqual(job['policy'], self.policy) + self.assertEqual(job['local_dev'], self.local_dev) + self.assertEqual(job['device'], self.local_dev['device']) + { + object_reconstructor.SYNC: sync_jobs, + object_reconstructor.REVERT: revert_jobs, + }[job['job_type']].append(job) + self.assertEqual(1, len(sync_jobs)) + job = sync_jobs[0] + self.assertEqual(job['frag_index'], frag_index) + self.assertEqual(sorted(job['suffixes']), sorted(['123', 'abc'])) + self.assertEqual(len(job['sync_to']), 2) + self.assertEqual(set([n['index'] for n in job['sync_to']]), + set([(frag_index + 1) % ring.replicas, + (frag_index - 1) % ring.replicas])) + self.assertEqual(1, len(revert_jobs)) + job = revert_jobs[0] + self.assertEqual(job['frag_index'], other_frag_index) + self.assertEqual(job['suffixes'], ['456']) + self.assertEqual(len(job['sync_to']), 1) + self.assertEqual(job['sync_to'][0]['index'], other_frag_index) + + def test_build_jobs_revert_only_tombstones(self): + ring = self.policy.object_ring = FabricatedRing() + # find a partition for which we're a handoff + for partition in range(2 ** ring.part_power): + part_nodes = ring.get_part_nodes(partition) + if self.local_dev['id'] not in [n['id'] for n in part_nodes]: + break + else: + self.fail("the ring doesn't work: %r" % ring._replica2part2dev_id) + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + part_info = { + 'local_dev': self.local_dev, + 'policy': self.policy, + 'partition': partition, + 'part_path': part_path, + } + # we have no fragment index to hint the jobs where they belong + stub_hashes = { + '123': {None: 'hash'}, + 'abc': {None: 'hash'}, + } + with mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes)): + jobs = self.reconstructor.build_reconstruction_jobs(part_info) + self.assertEqual(len(jobs), 1) + job = jobs[0] + expected = { + 'job_type': object_reconstructor.REVERT, + 'frag_index': None, + 'suffixes': stub_hashes.keys(), + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': self.local_dev, + 'device': self.local_dev['device'], + } + self.assertEqual(ring.replica_count, len(job['sync_to'])) + for k, v in expected.items(): + msg = 'expected %s != %s for %s' % ( + v, job[k], k) + self.assertEqual(v, job[k], msg) + + def test_get_suffix_delta(self): + # different + local_suff = {'123': {None: 'abc', 0: 'def'}} + remote_suff = {'456': {None: 'ghi', 0: 'jkl'}} + local_index = 0 + remote_index = 0 + suffs = self.reconstructor.get_suffix_delta(local_suff, + local_index, + remote_suff, + remote_index) + self.assertEqual(suffs, ['123']) + + # now the same + remote_suff = {'123': {None: 'abc', 0: 'def'}} + suffs = self.reconstructor.get_suffix_delta(local_suff, + local_index, + remote_suff, + remote_index) + self.assertEqual(suffs, []) + + # now with a mis-matched None key (missing durable) + remote_suff = {'123': {None: 'ghi', 0: 'def'}} + suffs = self.reconstructor.get_suffix_delta(local_suff, + local_index, + remote_suff, + remote_index) + self.assertEqual(suffs, ['123']) + + # now with bogus local index + local_suff = {'123': {None: 'abc', 99: 'def'}} + remote_suff = {'456': {None: 'ghi', 0: 'jkl'}} + suffs = self.reconstructor.get_suffix_delta(local_suff, + local_index, + remote_suff, + remote_index) + self.assertEqual(suffs, ['123']) + + def test_process_job_primary_in_sync(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [n for n in self.policy.object_ring.devs + if n != self.local_dev][:2] + # setup left and right hashes + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + left_index = sync_to[0]['index'] = (frag_index - 1) % replicas + left_hashes = { + '123': {left_index: 'hash', None: 'hash'}, + 'abc': {left_index: 'hash', None: 'hash'}, + } + right_index = sync_to[1]['index'] = (frag_index + 1) % replicas + right_hashes = { + '123': {right_index: 'hash', None: 'hash'}, + 'abc': {right_index: 'hash', None: 'hash'}, + } + partition = 0 + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.SYNC, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + responses = [(200, pickle.dumps(hashes)) for hashes in ( + left_hashes, right_hashes)] + codes, body_iter = zip(*responses) + + ssync_calls = [] + + with nested( + mock_ssync_sender(ssync_calls), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*codes, body_iter=body_iter) as request_log: + self.reconstructor.process_job(job) + + expected_suffix_calls = set([ + ('10.0.0.1', '/sdb/0'), + ('10.0.0.2', '/sdc/0'), + ]) + self.assertEqual(expected_suffix_calls, + set((r['ip'], r['path']) + for r in request_log.requests)) + + self.assertEqual(len(ssync_calls), 0) + + def test_process_job_primary_not_in_sync(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [n for n in self.policy.object_ring.devs + if n != self.local_dev][:2] + # setup left and right hashes + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + sync_to[0]['index'] = (frag_index - 1) % replicas + left_hashes = {} + sync_to[1]['index'] = (frag_index + 1) % replicas + right_hashes = {} + + partition = 0 + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.SYNC, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + responses = [(200, pickle.dumps(hashes)) for hashes in ( + left_hashes, left_hashes, right_hashes, right_hashes)] + codes, body_iter = zip(*responses) + + ssync_calls = [] + with nested( + mock_ssync_sender(ssync_calls), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*codes, body_iter=body_iter) as request_log: + self.reconstructor.process_job(job) + + expected_suffix_calls = set([ + ('10.0.0.1', '/sdb/0'), + ('10.0.0.1', '/sdb/0/123-abc'), + ('10.0.0.2', '/sdc/0'), + ('10.0.0.2', '/sdc/0/123-abc'), + ]) + self.assertEqual(expected_suffix_calls, + set((r['ip'], r['path']) + for r in request_log.requests)) + + expected_ssync_calls = sorted([ + ('10.0.0.1', 0, set(['123', 'abc'])), + ('10.0.0.2', 0, set(['123', 'abc'])), + ]) + self.assertEqual(expected_ssync_calls, sorted(( + c['node']['ip'], + c['job']['partition'], + set(c['suffixes']), + ) for c in ssync_calls)) + + def test_process_job_sync_missing_durable(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [n for n in self.policy.object_ring.devs + if n != self.local_dev][:2] + # setup left and right hashes + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + # left hand side is in sync + left_index = sync_to[0]['index'] = (frag_index - 1) % replicas + left_hashes = { + '123': {left_index: 'hash', None: 'hash'}, + 'abc': {left_index: 'hash', None: 'hash'}, + } + # right hand side has fragment, but no durable (None key is whack) + right_index = sync_to[1]['index'] = (frag_index + 1) % replicas + right_hashes = { + '123': {right_index: 'hash', None: 'hash'}, + 'abc': {right_index: 'hash', None: 'different-because-durable'}, + } + + partition = 0 + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.SYNC, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + responses = [(200, pickle.dumps(hashes)) for hashes in ( + left_hashes, right_hashes, right_hashes)] + codes, body_iter = zip(*responses) + + ssync_calls = [] + with nested( + mock_ssync_sender(ssync_calls), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*codes, body_iter=body_iter) as request_log: + self.reconstructor.process_job(job) + + expected_suffix_calls = set([ + ('10.0.0.1', '/sdb/0'), + ('10.0.0.2', '/sdc/0'), + ('10.0.0.2', '/sdc/0/abc'), + ]) + self.assertEqual(expected_suffix_calls, + set((r['ip'], r['path']) + for r in request_log.requests)) + + expected_ssync_calls = sorted([ + ('10.0.0.2', 0, ['abc']), + ]) + self.assertEqual(expected_ssync_calls, sorted(( + c['node']['ip'], + c['job']['partition'], + c['suffixes'], + ) for c in ssync_calls)) + + def test_process_job_primary_some_in_sync(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [n for n in self.policy.object_ring.devs + if n != self.local_dev][:2] + # setup left and right hashes + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + left_index = sync_to[0]['index'] = (frag_index - 1) % replicas + left_hashes = { + '123': {left_index: 'hashX', None: 'hash'}, + 'abc': {left_index: 'hash', None: 'hash'}, + } + right_index = sync_to[1]['index'] = (frag_index + 1) % replicas + right_hashes = { + '123': {right_index: 'hash', None: 'hash'}, + } + partition = 0 + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.SYNC, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + responses = [(200, pickle.dumps(hashes)) for hashes in ( + left_hashes, left_hashes, right_hashes, right_hashes)] + codes, body_iter = zip(*responses) + + ssync_calls = [] + + with nested( + mock_ssync_sender(ssync_calls), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*codes, body_iter=body_iter) as request_log: + self.reconstructor.process_job(job) + + expected_suffix_calls = set([ + ('10.0.0.1', '/sdb/0'), + ('10.0.0.1', '/sdb/0/123'), + ('10.0.0.2', '/sdc/0'), + ('10.0.0.2', '/sdc/0/abc'), + ]) + self.assertEqual(expected_suffix_calls, + set((r['ip'], r['path']) + for r in request_log.requests)) + + self.assertEqual(len(ssync_calls), 2) + self.assertEqual(set(c['node']['index'] for c in ssync_calls), + set([left_index, right_index])) + for call in ssync_calls: + if call['node']['index'] == left_index: + self.assertEqual(call['suffixes'], ['123']) + elif call['node']['index'] == right_index: + self.assertEqual(call['suffixes'], ['abc']) + else: + self.fail('unexpected call %r' % call) + + def test_process_job_primary_down(self): + replicas = self.policy.object_ring.replicas + partition = 0 + frag_index = random.randint(0, replicas - 1) + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + + part_nodes = self.policy.object_ring.get_part_nodes(partition) + sync_to = part_nodes[:2] + + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.SYNC, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'device': self.local_dev['device'], + 'local_dev': self.local_dev, + } + + non_local = {'called': 0} + + def ssync_response_callback(*args): + # in this test, ssync fails on the first (primary sync_to) node + if non_local['called'] >= 1: + return True, {} + non_local['called'] += 1 + return False, {} + + expected_suffix_calls = set() + for node in part_nodes[:3]: + expected_suffix_calls.update([ + (node['replication_ip'], '/%s/0' % node['device']), + (node['replication_ip'], '/%s/0/123-abc' % node['device']), + ]) + + ssync_calls = [] + with nested( + mock_ssync_sender(ssync_calls, + response_callback=ssync_response_callback), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*[200] * len(expected_suffix_calls), + body=pickle.dumps({})) as request_log: + self.reconstructor.process_job(job) + + found_suffix_calls = set((r['ip'], r['path']) + for r in request_log.requests) + self.assertEqual(expected_suffix_calls, found_suffix_calls) + + expected_ssync_calls = sorted([ + ('10.0.0.0', 0, set(['123', 'abc'])), + ('10.0.0.1', 0, set(['123', 'abc'])), + ('10.0.0.2', 0, set(['123', 'abc'])), + ]) + found_ssync_calls = sorted(( + c['node']['ip'], + c['job']['partition'], + set(c['suffixes']), + ) for c in ssync_calls) + self.assertEqual(expected_ssync_calls, found_ssync_calls) + + def test_process_job_suffix_call_errors(self): + replicas = self.policy.object_ring.replicas + partition = 0 + frag_index = random.randint(0, replicas - 1) + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + + part_nodes = self.policy.object_ring.get_part_nodes(partition) + sync_to = part_nodes[:2] + + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.SYNC, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'device': self.local_dev['device'], + 'local_dev': self.local_dev, + } + + expected_suffix_calls = set(( + node['replication_ip'], '/%s/0' % node['device'] + ) for node in part_nodes) + + possible_errors = [404, 507, Timeout(), Exception('kaboom!')] + codes = [random.choice(possible_errors) + for r in expected_suffix_calls] + + ssync_calls = [] + with nested( + mock_ssync_sender(ssync_calls), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*codes) as request_log: + self.reconstructor.process_job(job) + + found_suffix_calls = set((r['ip'], r['path']) + for r in request_log.requests) + self.assertEqual(expected_suffix_calls, found_suffix_calls) + + self.assertFalse(ssync_calls) + + def test_process_job_handoff(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [random.choice([n for n in self.policy.object_ring.devs + if n != self.local_dev])] + sync_to[0]['index'] = frag_index + + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + partition = 0 + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.REVERT, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + ssync_calls = [] + with nested( + mock_ssync_sender(ssync_calls), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(200, body=pickle.dumps({})) as request_log: + self.reconstructor.process_job(job) + + expected_suffix_calls = set([ + (sync_to[0]['ip'], '/%s/0/123-abc' % sync_to[0]['device']), + ]) + found_suffix_calls = set((r['ip'], r['path']) + for r in request_log.requests) + self.assertEqual(expected_suffix_calls, found_suffix_calls) + + self.assertEqual(len(ssync_calls), 1) + call = ssync_calls[0] + self.assertEqual(call['node'], sync_to[0]) + self.assertEqual(set(call['suffixes']), set(['123', 'abc'])) + + def test_process_job_revert_to_handoff(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [random.choice([n for n in self.policy.object_ring.devs + if n != self.local_dev])] + sync_to[0]['index'] = frag_index + partition = 0 + handoff = next(self.policy.object_ring.get_more_nodes(partition)) + + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.REVERT, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + non_local = {'called': 0} + + def ssync_response_callback(*args): + # in this test, ssync fails on the first (primary sync_to) node + if non_local['called'] >= 1: + return True, {} + non_local['called'] += 1 + return False, {} + + expected_suffix_calls = set([ + (node['replication_ip'], '/%s/0/123-abc' % node['device']) + for node in (sync_to[0], handoff) + ]) + + ssync_calls = [] + with nested( + mock_ssync_sender(ssync_calls, + response_callback=ssync_response_callback), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*[200] * len(expected_suffix_calls), + body=pickle.dumps({})) as request_log: + self.reconstructor.process_job(job) + + found_suffix_calls = set((r['ip'], r['path']) + for r in request_log.requests) + self.assertEqual(expected_suffix_calls, found_suffix_calls) + + self.assertEqual(len(ssync_calls), len(expected_suffix_calls)) + call = ssync_calls[0] + self.assertEqual(call['node'], sync_to[0]) + self.assertEqual(set(call['suffixes']), set(['123', 'abc'])) + + def test_process_job_revert_is_handoff(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [random.choice([n for n in self.policy.object_ring.devs + if n != self.local_dev])] + sync_to[0]['index'] = frag_index + partition = 0 + handoff_nodes = list(self.policy.object_ring.get_more_nodes(partition)) + + stub_hashes = { + '123': {frag_index: 'hash', None: 'hash'}, + 'abc': {frag_index: 'hash', None: 'hash'}, + } + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + job = { + 'job_type': object_reconstructor.REVERT, + 'frag_index': frag_index, + 'suffixes': stub_hashes.keys(), + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': stub_hashes, + 'policy': self.policy, + 'local_dev': handoff_nodes[-1], + } + + def ssync_response_callback(*args): + # in this test ssync always fails, until we encounter ourselves in + # the list of possible handoff's to sync to + return False, {} + + expected_suffix_calls = set([ + (sync_to[0]['replication_ip'], + '/%s/0/123-abc' % sync_to[0]['device']) + ] + [ + (node['replication_ip'], '/%s/0/123-abc' % node['device']) + for node in handoff_nodes[:-1] + ]) + + ssync_calls = [] + with nested( + mock_ssync_sender(ssync_calls, + response_callback=ssync_response_callback), + mock.patch('swift.obj.diskfile.ECDiskFileManager._get_hashes', + return_value=(None, stub_hashes))): + with mocked_http_conn(*[200] * len(expected_suffix_calls), + body=pickle.dumps({})) as request_log: + self.reconstructor.process_job(job) + + found_suffix_calls = set((r['ip'], r['path']) + for r in request_log.requests) + self.assertEqual(expected_suffix_calls, found_suffix_calls) + + # this is ssync call to primary (which fails) plus the ssync call to + # all of the handoffs (except the last one - which is the local_dev) + self.assertEqual(len(ssync_calls), len(handoff_nodes)) + call = ssync_calls[0] + self.assertEqual(call['node'], sync_to[0]) + self.assertEqual(set(call['suffixes']), set(['123', 'abc'])) + + def test_process_job_revert_cleanup(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [random.choice([n for n in self.policy.object_ring.devs + if n != self.local_dev])] + sync_to[0]['index'] = frag_index + partition = 0 + + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + os.makedirs(part_path) + df_mgr = self.reconstructor._df_router[self.policy] + df = df_mgr.get_diskfile(self.local_dev['device'], partition, 'a', + 'c', 'data-obj', policy=self.policy) + ts = self.ts() + with df.create() as writer: + test_data = 'test data' + writer.write(test_data) + metadata = { + 'X-Timestamp': ts.internal, + 'Content-Length': len(test_data), + 'Etag': md5(test_data).hexdigest(), + 'X-Object-Sysmeta-Ec-Frag-Index': frag_index, + } + writer.put(metadata) + writer.commit(ts) + + ohash = os.path.basename(df._datadir) + suffix = os.path.basename(os.path.dirname(df._datadir)) + + job = { + 'job_type': object_reconstructor.REVERT, + 'frag_index': frag_index, + 'suffixes': [suffix], + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': {}, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + def ssync_response_callback(*args): + return True, {ohash: ts} + + ssync_calls = [] + with mock_ssync_sender(ssync_calls, + response_callback=ssync_response_callback): + with mocked_http_conn(200, body=pickle.dumps({})) as request_log: + self.reconstructor.process_job(job) + + self.assertEqual([ + (sync_to[0]['replication_ip'], '/%s/0/%s' % ( + sync_to[0]['device'], suffix)), + ], [ + (r['ip'], r['path']) for r in request_log.requests + ]) + # hashpath is still there, but only the durable remains + files = os.listdir(df._datadir) + self.assertEqual(1, len(files)) + self.assertTrue(files[0].endswith('.durable')) + + # and more to the point, the next suffix recalc will clean it up + df_mgr = self.reconstructor._df_router[self.policy] + df_mgr.get_hashes(self.local_dev['device'], str(partition), [], + self.policy) + self.assertFalse(os.access(df._datadir, os.F_OK)) + + def test_process_job_revert_cleanup_tombstone(self): + replicas = self.policy.object_ring.replicas + frag_index = random.randint(0, replicas - 1) + sync_to = [random.choice([n for n in self.policy.object_ring.devs + if n != self.local_dev])] + sync_to[0]['index'] = frag_index + partition = 0 + + part_path = os.path.join(self.devices, self.local_dev['device'], + diskfile.get_data_dir(self.policy), + str(partition)) + os.makedirs(part_path) + df_mgr = self.reconstructor._df_router[self.policy] + df = df_mgr.get_diskfile(self.local_dev['device'], partition, 'a', + 'c', 'data-obj', policy=self.policy) + ts = self.ts() + df.delete(ts) + + ohash = os.path.basename(df._datadir) + suffix = os.path.basename(os.path.dirname(df._datadir)) + + job = { + 'job_type': object_reconstructor.REVERT, + 'frag_index': frag_index, + 'suffixes': [suffix], + 'sync_to': sync_to, + 'partition': partition, + 'path': part_path, + 'hashes': {}, + 'policy': self.policy, + 'local_dev': self.local_dev, + } + + def ssync_response_callback(*args): + return True, {ohash: ts} + + ssync_calls = [] + with mock_ssync_sender(ssync_calls, + response_callback=ssync_response_callback): + with mocked_http_conn(200, body=pickle.dumps({})) as request_log: + self.reconstructor.process_job(job) + + self.assertEqual([ + (sync_to[0]['replication_ip'], '/%s/0/%s' % ( + sync_to[0]['device'], suffix)), + ], [ + (r['ip'], r['path']) for r in request_log.requests + ]) + # hashpath is still there, but it's empty + self.assertEqual([], os.listdir(df._datadir)) + + def test_reconstruct_fa_no_errors(self): + job = { + 'partition': 0, + 'policy': self.policy, + } + part_nodes = self.policy.object_ring.get_part_nodes(0) + node = part_nodes[1] + metadata = { + 'name': '/a/c/o', + 'Content-Length': 0, + 'ETag': 'etag', + } + + test_data = ('rebuild' * self.policy.ec_segment_size)[:-777] + etag = md5(test_data).hexdigest() + ec_archive_bodies = make_ec_archive_bodies(self.policy, test_data) + + broken_body = ec_archive_bodies.pop(1) + + responses = list((200, body) for body in ec_archive_bodies) + headers = {'X-Object-Sysmeta-Ec-Etag': etag} + codes, body_iter = zip(*responses) + with mocked_http_conn(*codes, body_iter=body_iter, headers=headers): + df = self.reconstructor.reconstruct_fa( + job, node, metadata) + fixed_body = ''.join(df.reader()) + self.assertEqual(len(fixed_body), len(broken_body)) + self.assertEqual(md5(fixed_body).hexdigest(), + md5(broken_body).hexdigest()) + + def test_reconstruct_fa_errors_works(self): + job = { + 'partition': 0, + 'policy': self.policy, + } + part_nodes = self.policy.object_ring.get_part_nodes(0) + node = part_nodes[4] + metadata = { + 'name': '/a/c/o', + 'Content-Length': 0, + 'ETag': 'etag', + } + + test_data = ('rebuild' * self.policy.ec_segment_size)[:-777] + etag = md5(test_data).hexdigest() + ec_archive_bodies = make_ec_archive_bodies(self.policy, test_data) + + broken_body = ec_archive_bodies.pop(4) + + base_responses = list((200, body) for body in ec_archive_bodies) + # since we're already missing a fragment a +2 scheme can only support + # one additional failure at a time + for error in (Timeout(), 404, Exception('kaboom!')): + responses = list(base_responses) + error_index = random.randint(0, len(responses) - 1) + responses[error_index] = (error, '') + headers = {'X-Object-Sysmeta-Ec-Etag': etag} + codes, body_iter = zip(*responses) + with mocked_http_conn(*codes, body_iter=body_iter, + headers=headers): + df = self.reconstructor.reconstruct_fa( + job, node, dict(metadata)) + fixed_body = ''.join(df.reader()) + self.assertEqual(len(fixed_body), len(broken_body)) + self.assertEqual(md5(fixed_body).hexdigest(), + md5(broken_body).hexdigest()) + + def test_reconstruct_fa_errors_fails(self): + job = { + 'partition': 0, + 'policy': self.policy, + } + part_nodes = self.policy.object_ring.get_part_nodes(0) + node = part_nodes[1] + policy = self.policy + metadata = { + 'name': '/a/c/o', + 'Content-Length': 0, + 'ETag': 'etag', + } + + possible_errors = [404, Timeout(), Exception('kaboom!')] + codes = [random.choice(possible_errors) for i in + range(policy.object_ring.replicas - 1)] + with mocked_http_conn(*codes): + self.assertRaises(DiskFileError, self.reconstructor.reconstruct_fa, + job, node, metadata) + + def test_reconstruct_fa_with_mixed_old_etag(self): + job = { + 'partition': 0, + 'policy': self.policy, + } + part_nodes = self.policy.object_ring.get_part_nodes(0) + node = part_nodes[1] + metadata = { + 'name': '/a/c/o', + 'Content-Length': 0, + 'ETag': 'etag', + } + + test_data = ('rebuild' * self.policy.ec_segment_size)[:-777] + etag = md5(test_data).hexdigest() + ec_archive_bodies = make_ec_archive_bodies(self.policy, test_data) + + broken_body = ec_archive_bodies.pop(1) + + ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) + # bad response + bad_response = (200, '', { + 'X-Object-Sysmeta-Ec-Etag': 'some garbage', + 'X-Backend-Timestamp': next(ts).internal, + }) + + # good responses + headers = { + 'X-Object-Sysmeta-Ec-Etag': etag, + 'X-Backend-Timestamp': next(ts).internal + } + responses = [(200, body, headers) + for body in ec_archive_bodies] + # mixed together + error_index = random.randint(0, len(responses) - 2) + responses[error_index] = bad_response + codes, body_iter, headers = zip(*responses) + with mocked_http_conn(*codes, body_iter=body_iter, headers=headers): + df = self.reconstructor.reconstruct_fa( + job, node, metadata) + fixed_body = ''.join(df.reader()) + self.assertEqual(len(fixed_body), len(broken_body)) + self.assertEqual(md5(fixed_body).hexdigest(), + md5(broken_body).hexdigest()) + + def test_reconstruct_fa_with_mixed_new_etag(self): + job = { + 'partition': 0, + 'policy': self.policy, + } + part_nodes = self.policy.object_ring.get_part_nodes(0) + node = part_nodes[1] + metadata = { + 'name': '/a/c/o', + 'Content-Length': 0, + 'ETag': 'etag', + } + + test_data = ('rebuild' * self.policy.ec_segment_size)[:-777] + etag = md5(test_data).hexdigest() + ec_archive_bodies = make_ec_archive_bodies(self.policy, test_data) + + broken_body = ec_archive_bodies.pop(1) + + ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) + # good responses + headers = { + 'X-Object-Sysmeta-Ec-Etag': etag, + 'X-Backend-Timestamp': next(ts).internal + } + responses = [(200, body, headers) + for body in ec_archive_bodies] + codes, body_iter, headers = zip(*responses) + + # sanity check before negative test + with mocked_http_conn(*codes, body_iter=body_iter, headers=headers): + df = self.reconstructor.reconstruct_fa( + job, node, dict(metadata)) + fixed_body = ''.join(df.reader()) + self.assertEqual(len(fixed_body), len(broken_body)) + self.assertEqual(md5(fixed_body).hexdigest(), + md5(broken_body).hexdigest()) + + # one newer etag can spoil the bunch + new_response = (200, '', { + 'X-Object-Sysmeta-Ec-Etag': 'some garbage', + 'X-Backend-Timestamp': next(ts).internal, + }) + new_index = random.randint(0, len(responses) - self.policy.ec_nparity) + responses[new_index] = new_response + codes, body_iter, headers = zip(*responses) + with mocked_http_conn(*codes, body_iter=body_iter, headers=headers): + self.assertRaises(DiskFileError, self.reconstructor.reconstruct_fa, + job, node, dict(metadata)) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index ab89e4925..f169e52dd 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -27,7 +27,7 @@ from errno import ENOENT, ENOTEMPTY, ENOTDIR from eventlet.green import subprocess from eventlet import Timeout, tpool -from test.unit import FakeLogger, debug_logger, patch_policies +from test.unit import debug_logger, patch_policies from swift.common import utils from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ storage_directory @@ -173,9 +173,9 @@ class TestObjectReplicator(unittest.TestCase): os.mkdir(self.devices) os.mkdir(os.path.join(self.devices, 'sda')) self.objects = os.path.join(self.devices, 'sda', - diskfile.get_data_dir(0)) + diskfile.get_data_dir(POLICIES[0])) self.objects_1 = os.path.join(self.devices, 'sda', - diskfile.get_data_dir(1)) + diskfile.get_data_dir(POLICIES[1])) os.mkdir(self.objects) os.mkdir(self.objects_1) self.parts = {} @@ -190,7 +190,7 @@ class TestObjectReplicator(unittest.TestCase): swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1', sync_method='rsync') self.replicator = object_replicator.ObjectReplicator(self.conf) - self.replicator.logger = FakeLogger() + self.logger = self.replicator.logger = debug_logger('test-replicator') self.df_mgr = diskfile.DiskFileManager(self.conf, self.replicator.logger) @@ -205,7 +205,7 @@ class TestObjectReplicator(unittest.TestCase): object_replicator.http_connect = mock_http_connect(200) cur_part = '0' df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o', - policy_idx=0) + policy=POLICIES[0]) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -216,7 +216,7 @@ class TestObjectReplicator(unittest.TestCase): data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] - ring = replicator.get_object_ring(0) + ring = replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] @@ -239,7 +239,7 @@ class TestObjectReplicator(unittest.TestCase): object_replicator.http_connect = mock_http_connect(200) cur_part = '0' df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o', - policy_idx=1) + policy=POLICIES[1]) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -250,7 +250,7 @@ class TestObjectReplicator(unittest.TestCase): data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects_1, cur_part, data_dir) process_arg_checker = [] - ring = replicator.get_object_ring(1) + ring = replicator.load_object_ring(POLICIES[1]) nodes = [node for node in ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] @@ -266,7 +266,7 @@ class TestObjectReplicator(unittest.TestCase): def test_check_ring(self): for pol in POLICIES: - obj_ring = self.replicator.get_object_ring(pol.idx) + obj_ring = self.replicator.load_object_ring(pol) self.assertTrue(self.replicator.check_ring(obj_ring)) orig_check = self.replicator.next_check self.replicator.next_check = orig_check - 30 @@ -280,29 +280,27 @@ class TestObjectReplicator(unittest.TestCase): def test_collect_jobs_mkdirs_error(self): + non_local = {} + def blowup_mkdirs(path): + non_local['path'] = path raise OSError('Ow!') with mock.patch.object(object_replicator, 'mkdirs', blowup_mkdirs): rmtree(self.objects, ignore_errors=1) object_replicator.mkdirs = blowup_mkdirs self.replicator.collect_jobs() - self.assertTrue('exception' in self.replicator.logger.log_dict) - self.assertEquals( - len(self.replicator.logger.log_dict['exception']), 1) - exc_args, exc_kwargs, exc_str = \ - self.replicator.logger.log_dict['exception'][0] - self.assertEquals(len(exc_args), 1) - self.assertTrue(exc_args[0].startswith('ERROR creating ')) - self.assertEquals(exc_kwargs, {}) - self.assertEquals(exc_str, 'Ow!') + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'ERROR creating %s: ' % non_local['path']]) + log_args, log_kwargs = self.logger.log_dict['error'][0] + self.assertEqual(str(log_kwargs['exc_info'][1]), 'Ow!') def test_collect_jobs(self): jobs = self.replicator.collect_jobs() jobs_to_delete = [j for j in jobs if j['delete']] jobs_by_pol_part = {} for job in jobs: - jobs_by_pol_part[str(job['policy_idx']) + job['partition']] = job + jobs_by_pol_part[str(int(job['policy'])) + job['partition']] = job self.assertEquals(len(jobs_to_delete), 2) self.assertTrue('1', jobs_to_delete[0]['partition']) self.assertEquals( @@ -383,19 +381,19 @@ class TestObjectReplicator(unittest.TestCase): self.assertFalse(os.path.exists(pol_0_part_1_path)) self.assertFalse(os.path.exists(pol_1_part_1_path)) - - logged_warnings = sorted(self.replicator.logger.log_dict['warning']) - self.assertEquals( - (('Removing partition directory which was a file: %s', - pol_1_part_1_path), {}), logged_warnings[0]) - self.assertEquals( - (('Removing partition directory which was a file: %s', - pol_0_part_1_path), {}), logged_warnings[1]) + self.assertEqual( + sorted(self.logger.get_lines_for_level('warning')), [ + ('Removing partition directory which was a file: %s' + % pol_1_part_1_path), + ('Removing partition directory which was a file: %s' + % pol_0_part_1_path), + ]) def test_delete_partition(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -407,7 +405,7 @@ class TestObjectReplicator(unittest.TestCase): whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) - ring = self.replicator.get_object_ring(0) + ring = self.replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] @@ -424,7 +422,8 @@ class TestObjectReplicator(unittest.TestCase): self.replicator.conf.pop('sync_method') with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -436,7 +435,7 @@ class TestObjectReplicator(unittest.TestCase): whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) - ring = self.replicator.get_object_ring(0) + ring = self.replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] @@ -473,10 +472,11 @@ class TestObjectReplicator(unittest.TestCase): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) - f = open(os.path.join(df._datadir, - normalize_timestamp(time.time()) + '.data'), + ts = normalize_timestamp(time.time()) + f = open(os.path.join(df._datadir, ts + '.data'), 'wb') f.write('1234567890') f.close() @@ -487,7 +487,7 @@ class TestObjectReplicator(unittest.TestCase): self.assertTrue(os.access(part_path, os.F_OK)) def _fake_ssync(node, job, suffixes, **kwargs): - return True, set([ohash]) + return True, {ohash: ts} self.replicator.sync_method = _fake_ssync self.replicator.replicate() @@ -499,7 +499,7 @@ class TestObjectReplicator(unittest.TestCase): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', - policy_idx=1) + policy=POLICIES[1]) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -511,7 +511,7 @@ class TestObjectReplicator(unittest.TestCase): whole_path_from = os.path.join(self.objects_1, '1', data_dir) part_path = os.path.join(self.objects_1, '1') self.assertTrue(os.access(part_path, os.F_OK)) - ring = self.replicator.get_object_ring(1) + ring = self.replicator.load_object_ring(POLICIES[1]) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] @@ -527,7 +527,8 @@ class TestObjectReplicator(unittest.TestCase): def test_delete_partition_with_failures(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -539,7 +540,7 @@ class TestObjectReplicator(unittest.TestCase): whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) - ring = self.replicator.get_object_ring(0) + ring = self.replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] @@ -562,7 +563,8 @@ class TestObjectReplicator(unittest.TestCase): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.handoff_delete = 2 - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -574,7 +576,7 @@ class TestObjectReplicator(unittest.TestCase): whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) - ring = self.replicator.get_object_ring(0) + ring = self.replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] @@ -596,7 +598,8 @@ class TestObjectReplicator(unittest.TestCase): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.handoff_delete = 2 - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -608,7 +611,7 @@ class TestObjectReplicator(unittest.TestCase): whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) - ring = self.replicator.get_object_ring(0) + ring = self.replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] @@ -630,7 +633,8 @@ class TestObjectReplicator(unittest.TestCase): def test_delete_partition_with_handoff_delete_fail_in_other_region(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -642,7 +646,7 @@ class TestObjectReplicator(unittest.TestCase): whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) - ring = self.replicator.get_object_ring(0) + ring = self.replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] @@ -662,7 +666,8 @@ class TestObjectReplicator(unittest.TestCase): self.assertTrue(os.access(part_path, os.F_OK)) def test_delete_partition_override_params(self): - df = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) @@ -675,9 +680,10 @@ class TestObjectReplicator(unittest.TestCase): self.assertFalse(os.access(part_path, os.F_OK)) def test_delete_policy_override_params(self): - df0 = self.df_mgr.get_diskfile('sda', '99', 'a', 'c', 'o') + df0 = self.df_mgr.get_diskfile('sda', '99', 'a', 'c', 'o', + policy=POLICIES.legacy) df1 = self.df_mgr.get_diskfile('sda', '99', 'a', 'c', 'o', - policy_idx=1) + policy=POLICIES[1]) mkdirs(df0._datadir) mkdirs(df1._datadir) @@ -698,10 +704,11 @@ class TestObjectReplicator(unittest.TestCase): def test_delete_partition_ssync(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) - f = open(os.path.join(df._datadir, - normalize_timestamp(time.time()) + '.data'), + ts = normalize_timestamp(time.time()) + f = open(os.path.join(df._datadir, ts + '.data'), 'wb') f.write('0') f.close() @@ -716,14 +723,14 @@ class TestObjectReplicator(unittest.TestCase): def _fake_ssync(node, job, suffixes, **kwargs): success = True - ret_val = [whole_path_from] + ret_val = {ohash: ts} if self.call_nums == 2: # ssync should return (True, []) only when the second # candidate node has not get the replica yet. success = False - ret_val = [] + ret_val = {} self.call_nums += 1 - return success, set(ret_val) + return success, ret_val self.replicator.sync_method = _fake_ssync self.replicator.replicate() @@ -746,11 +753,11 @@ class TestObjectReplicator(unittest.TestCase): def test_delete_partition_ssync_with_sync_failure(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) + ts = normalize_timestamp(time.time()) mkdirs(df._datadir) - f = open(os.path.join(df._datadir, - normalize_timestamp(time.time()) + '.data'), - 'wb') + f = open(os.path.join(df._datadir, ts + '.data'), 'wb') f.write('0') f.close() ohash = hash_path('a', 'c', 'o') @@ -763,14 +770,14 @@ class TestObjectReplicator(unittest.TestCase): def _fake_ssync(node, job, suffixes, **kwags): success = False - ret_val = [] + ret_val = {} if self.call_nums == 2: # ssync should return (True, []) only when the second # candidate node has not get the replica yet. success = True - ret_val = [whole_path_from] + ret_val = {ohash: ts} self.call_nums += 1 - return success, set(ret_val) + return success, ret_val self.replicator.sync_method = _fake_ssync self.replicator.replicate() @@ -794,11 +801,11 @@ class TestObjectReplicator(unittest.TestCase): self.replicator.logger = debug_logger('test-replicator') with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) - f = open(os.path.join(df._datadir, - normalize_timestamp(time.time()) + '.data'), - 'wb') + ts = normalize_timestamp(time.time()) + f = open(os.path.join(df._datadir, ts + '.data'), 'wb') f.write('0') f.close() ohash = hash_path('a', 'c', 'o') @@ -809,16 +816,16 @@ class TestObjectReplicator(unittest.TestCase): self.call_nums = 0 self.conf['sync_method'] = 'ssync' - in_sync_objs = [] + in_sync_objs = {} def _fake_ssync(node, job, suffixes, remote_check_objs=None): self.call_nums += 1 if remote_check_objs is None: # sync job - ret_val = [whole_path_from] + ret_val = {ohash: ts} else: ret_val = in_sync_objs - return True, set(ret_val) + return True, ret_val self.replicator.sync_method = _fake_ssync self.replicator.replicate() @@ -833,12 +840,13 @@ class TestObjectReplicator(unittest.TestCase): def test_delete_partition_ssync_with_cleanup_failure(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): - self.replicator.logger = mock_logger = mock.MagicMock() - df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') + self.replicator.logger = mock_logger = \ + debug_logger('test-replicator') + df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) - f = open(os.path.join(df._datadir, - normalize_timestamp(time.time()) + '.data'), - 'wb') + ts = normalize_timestamp(time.time()) + f = open(os.path.join(df._datadir, ts + '.data'), 'wb') f.write('0') f.close() ohash = hash_path('a', 'c', 'o') @@ -852,14 +860,14 @@ class TestObjectReplicator(unittest.TestCase): def _fake_ssync(node, job, suffixes, **kwargs): success = True - ret_val = [whole_path_from] + ret_val = {ohash: ts} if self.call_nums == 2: # ssync should return (True, []) only when the second # candidate node has not get the replica yet. success = False - ret_val = [] + ret_val = {} self.call_nums += 1 - return success, set(ret_val) + return success, ret_val rmdir_func = os.rmdir @@ -886,7 +894,7 @@ class TestObjectReplicator(unittest.TestCase): with mock.patch('os.rmdir', raise_exception_rmdir(OSError, ENOENT)): self.replicator.replicate() - self.assertEquals(mock_logger.exception.call_count, 0) + self.assertFalse(mock_logger.get_lines_for_level('error')) self.assertFalse(os.access(whole_path_from, os.F_OK)) self.assertTrue(os.access(suffix_dir_path, os.F_OK)) self.assertTrue(os.access(part_path, os.F_OK)) @@ -895,7 +903,7 @@ class TestObjectReplicator(unittest.TestCase): with mock.patch('os.rmdir', raise_exception_rmdir(OSError, ENOTEMPTY)): self.replicator.replicate() - self.assertEquals(mock_logger.exception.call_count, 0) + self.assertFalse(mock_logger.get_lines_for_level('error')) self.assertFalse(os.access(whole_path_from, os.F_OK)) self.assertTrue(os.access(suffix_dir_path, os.F_OK)) self.assertTrue(os.access(part_path, os.F_OK)) @@ -904,7 +912,7 @@ class TestObjectReplicator(unittest.TestCase): with mock.patch('os.rmdir', raise_exception_rmdir(OSError, ENOTDIR)): self.replicator.replicate() - self.assertEquals(mock_logger.exception.call_count, 1) + self.assertEqual(len(mock_logger.get_lines_for_level('error')), 1) self.assertFalse(os.access(whole_path_from, os.F_OK)) self.assertTrue(os.access(suffix_dir_path, os.F_OK)) self.assertTrue(os.access(part_path, os.F_OK)) @@ -929,7 +937,8 @@ class TestObjectReplicator(unittest.TestCase): # Write some files into '1' and run replicate- they should be moved # to the other partitions and then node should get deleted. cur_part = '1' - df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -939,7 +948,7 @@ class TestObjectReplicator(unittest.TestCase): ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) - ring = replicator.get_object_ring(0) + ring = replicator.load_object_ring(POLICIES[0]) process_arg_checker = [] nodes = [node for node in ring.get_part_nodes(int(cur_part)) @@ -993,7 +1002,8 @@ class TestObjectReplicator(unittest.TestCase): # Write some files into '1' and run replicate- they should be moved # to the other partitions and then node should get deleted. cur_part = '1' - df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o') + df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o', + policy=POLICIES.legacy) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), @@ -1004,10 +1014,11 @@ class TestObjectReplicator(unittest.TestCase): data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] - ring = replicator.get_object_ring(0) + ring = replicator.load_object_ring(POLICIES[0]) nodes = [node for node in ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] + for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) @@ -1071,8 +1082,8 @@ class TestObjectReplicator(unittest.TestCase): expect = 'Error syncing partition' for job in jobs: set_default(self) - ring = self.replicator.get_object_ring(job['policy_idx']) - self.headers['X-Backend-Storage-Policy-Index'] = job['policy_idx'] + ring = job['policy'].object_ring + self.headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) self.replicator.update(job) self.assertTrue(error in mock_logger.error.call_args[0][0]) self.assertTrue(expect in mock_logger.exception.call_args[0][0]) @@ -1118,7 +1129,7 @@ class TestObjectReplicator(unittest.TestCase): for job in jobs: set_default(self) # limit local job to policy 0 for simplicity - if job['partition'] == '0' and job['policy_idx'] == 0: + if job['partition'] == '0' and int(job['policy']) == 0: local_job = job.copy() continue self.replicator.update(job) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 1823a9014..52a34347a 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -18,6 +18,7 @@ import cPickle as pickle import datetime +import json import errno import operator import os @@ -39,17 +40,19 @@ from eventlet.green import httplib from nose import SkipTest from swift import __version__ as swift_version +from swift.common.http import is_success from test.unit import FakeLogger, debug_logger, mocked_http_conn from test.unit import connect_tcp, readuntil2crlfs, patch_policies from swift.obj import server as object_server from swift.obj import diskfile -from swift.common import utils, storage_policy, bufferedhttp +from swift.common import utils, bufferedhttp from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ NullLogger, storage_directory, public, replication from swift.common import constraints -from swift.common.swob import Request, HeaderKeyDict +from swift.common.swob import Request, HeaderKeyDict, WsgiStringIO from swift.common.splice import splice -from swift.common.storage_policy import POLICIES +from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy, + POLICIES, EC_POLICY) from swift.common.exceptions import DiskFileDeviceUnavailable @@ -57,7 +60,14 @@ def mock_time(*args, **kwargs): return 5000.0 -@patch_policies +test_policies = [ + StoragePolicy(0, name='zero', is_default=True), + ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand', + ec_ndata=10, ec_nparity=4), +] + + +@patch_policies(test_policies) class TestObjectController(unittest.TestCase): """Test swift.obj.server.ObjectController""" @@ -68,15 +78,18 @@ class TestObjectController(unittest.TestCase): self.tmpdir = mkdtemp() self.testdir = os.path.join(self.tmpdir, 'tmp_test_object_server_ObjectController') - conf = {'devices': self.testdir, 'mount_check': 'false'} + mkdirs(os.path.join(self.testdir, 'sda1')) + self.conf = {'devices': self.testdir, 'mount_check': 'false'} self.object_controller = object_server.ObjectController( - conf, logger=debug_logger()) + self.conf, logger=debug_logger()) self.object_controller.bytes_per_sync = 1 self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) - self.df_mgr = diskfile.DiskFileManager(conf, + self.df_mgr = diskfile.DiskFileManager(self.conf, self.object_controller.logger) + self.logger = debug_logger('test-object-controller') + def tearDown(self): """Tear down for testing swift.object.server.ObjectController""" rmtree(self.tmpdir) @@ -84,7 +97,7 @@ class TestObjectController(unittest.TestCase): def _stage_tmp_dir(self, policy): mkdirs(os.path.join(self.testdir, 'sda1', - diskfile.get_tmp_dir(int(policy)))) + diskfile.get_tmp_dir(policy))) def check_all_api_methods(self, obj_name='o', alt_res=None): path = '/sda1/p/a/c/%s' % obj_name @@ -417,7 +430,8 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) - objfile = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') + objfile = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o', + policy=POLICIES.legacy) objfile.open() file_name = os.path.basename(objfile._data_file) with open(objfile._data_file) as fp: @@ -568,7 +582,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) @@ -603,7 +617,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) @@ -638,7 +652,7 @@ class TestObjectController(unittest.TestCase): self.assertEqual(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.data') self.assertTrue(os.path.isfile(objfile)) @@ -715,7 +729,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) @@ -729,6 +743,241 @@ class TestObjectController(unittest.TestCase): 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) + def test_PUT_etag_in_footer(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Transfer-Encoding': 'chunked', + 'Etag': 'other-etag', + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary'}, + environ={'REQUEST_METHOD': 'PUT'}) + + obj_etag = md5("obj data").hexdigest() + footer_meta = json.dumps({"Etag": obj_etag}) + footer_meta_cksum = md5(footer_meta).hexdigest() + + req.body = "\r\n".join(( + "--boundary", + "", + "obj data", + "--boundary", + "Content-MD5: " + footer_meta_cksum, + "", + footer_meta, + "--boundary--", + )) + req.headers.pop("Content-Length", None) + + resp = req.get_response(self.object_controller) + self.assertEqual(resp.etag, obj_etag) + self.assertEqual(resp.status_int, 201) + + objfile = os.path.join( + self.testdir, 'sda1', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', + hash_path('a', 'c', 'o')), + utils.Timestamp(timestamp).internal + '.data') + with open(objfile) as fh: + self.assertEqual(fh.read(), "obj data") + + def test_PUT_etag_in_footer_mismatch(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Transfer-Encoding': 'chunked', + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary'}, + environ={'REQUEST_METHOD': 'PUT'}) + + footer_meta = json.dumps({"Etag": md5("green").hexdigest()}) + footer_meta_cksum = md5(footer_meta).hexdigest() + + req.body = "\r\n".join(( + "--boundary", + "", + "blue", + "--boundary", + "Content-MD5: " + footer_meta_cksum, + "", + footer_meta, + "--boundary--", + )) + req.headers.pop("Content-Length", None) + + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 422) + + def test_PUT_meta_in_footer(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Transfer-Encoding': 'chunked', + 'X-Object-Meta-X': 'Z', + 'X-Object-Sysmeta-X': 'Z', + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary'}, + environ={'REQUEST_METHOD': 'PUT'}) + + footer_meta = json.dumps({ + 'X-Object-Meta-X': 'Y', + 'X-Object-Sysmeta-X': 'Y', + }) + footer_meta_cksum = md5(footer_meta).hexdigest() + + req.body = "\r\n".join(( + "--boundary", + "", + "stuff stuff stuff", + "--boundary", + "Content-MD5: " + footer_meta_cksum, + "", + footer_meta, + "--boundary--", + )) + req.headers.pop("Content-Length", None) + + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp}, + environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.headers.get('X-Object-Meta-X'), 'Y') + self.assertEqual(resp.headers.get('X-Object-Sysmeta-X'), 'Y') + + def test_PUT_missing_footer_checksum(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Transfer-Encoding': 'chunked', + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary'}, + environ={'REQUEST_METHOD': 'PUT'}) + + footer_meta = json.dumps({"Etag": md5("obj data").hexdigest()}) + + req.body = "\r\n".join(( + "--boundary", + "", + "obj data", + "--boundary", + # no Content-MD5 + "", + footer_meta, + "--boundary--", + )) + req.headers.pop("Content-Length", None) + + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 400) + + def test_PUT_bad_footer_checksum(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Transfer-Encoding': 'chunked', + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary'}, + environ={'REQUEST_METHOD': 'PUT'}) + + footer_meta = json.dumps({"Etag": md5("obj data").hexdigest()}) + bad_footer_meta_cksum = md5(footer_meta + "bad").hexdigest() + + req.body = "\r\n".join(( + "--boundary", + "", + "obj data", + "--boundary", + "Content-MD5: " + bad_footer_meta_cksum, + "", + footer_meta, + "--boundary--", + )) + req.headers.pop("Content-Length", None) + + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 422) + + def test_PUT_bad_footer_json(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Transfer-Encoding': 'chunked', + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary'}, + environ={'REQUEST_METHOD': 'PUT'}) + + footer_meta = "{{{[[{{[{[[{[{[[{{{[{{{{[[{{[{[" + footer_meta_cksum = md5(footer_meta).hexdigest() + + req.body = "\r\n".join(( + "--boundary", + "", + "obj data", + "--boundary", + "Content-MD5: " + footer_meta_cksum, + "", + footer_meta, + "--boundary--", + )) + req.headers.pop("Content-Length", None) + + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 400) + + def test_PUT_extra_mime_docs_ignored(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Transfer-Encoding': 'chunked', + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary'}, + environ={'REQUEST_METHOD': 'PUT'}) + + footer_meta = json.dumps({'X-Object-Meta-Mint': 'pepper'}) + footer_meta_cksum = md5(footer_meta).hexdigest() + + req.body = "\r\n".join(( + "--boundary", + "", + "obj data", + "--boundary", + "Content-MD5: " + footer_meta_cksum, + "", + footer_meta, + "--boundary", + "This-Document-Is-Useless: yes", + "", + "blah blah I take up space", + "--boundary--" + )) + req.headers.pop("Content-Length", None) + + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + # swob made this into a StringIO for us + wsgi_input = req.environ['wsgi.input'] + self.assertEqual(wsgi_input.tell(), len(wsgi_input.getvalue())) + def test_PUT_user_metadata_no_xattr(self): timestamp = normalize_timestamp(time()) req = Request.blank( @@ -768,7 +1017,7 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'Content-Length': '6'}) - req.environ['wsgi.input'] = StringIO('VERIFY') + req.environ['wsgi.input'] = WsgiStringIO('VERIFY') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 408) @@ -788,7 +1037,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assert_(os.path.isfile(objfile)) @@ -831,7 +1080,7 @@ class TestObjectController(unittest.TestCase): # original .data file metadata should be unchanged objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), timestamp1 + '.data') self.assert_(os.path.isfile(objfile)) @@ -849,7 +1098,7 @@ class TestObjectController(unittest.TestCase): # .meta file metadata should have only user meta items metafile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), timestamp2 + '.meta') self.assert_(os.path.isfile(metafile)) @@ -1017,6 +1266,40 @@ class TestObjectController(unittest.TestCase): finally: object_server.http_connect = old_http_connect + def test_PUT_durable_files(self): + for policy in POLICIES: + timestamp = utils.Timestamp(int(time())).internal + data_file_tail = '.data' + headers = {'X-Timestamp': timestamp, + 'Content-Length': '6', + 'Content-Type': 'application/octet-stream', + 'X-Backend-Storage-Policy-Index': int(policy)} + if policy.policy_type == EC_POLICY: + headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2' + data_file_tail = '#2.data' + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers=headers) + req.body = 'VERIFY' + resp = req.get_response(self.object_controller) + + self.assertEquals(resp.status_int, 201) + obj_dir = os.path.join( + self.testdir, 'sda1', + storage_directory(diskfile.get_data_dir(int(policy)), + 'p', hash_path('a', 'c', 'o'))) + data_file = os.path.join(obj_dir, timestamp) + data_file_tail + self.assertTrue(os.path.isfile(data_file), + 'Expected file %r not found in %r for policy %r' + % (data_file, os.listdir(obj_dir), int(policy))) + durable_file = os.path.join(obj_dir, timestamp) + '.durable' + if policy.policy_type == EC_POLICY: + self.assertTrue(os.path.isfile(durable_file)) + self.assertFalse(os.path.getsize(durable_file)) + else: + self.assertFalse(os.path.isfile(durable_file)) + rmtree(obj_dir) + def test_HEAD(self): # Test swift.obj.server.ObjectController.HEAD req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) @@ -1058,7 +1341,7 @@ class TestObjectController(unittest.TestCase): objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.data') os.unlink(objfile) @@ -1102,7 +1385,8 @@ class TestObjectController(unittest.TestCase): req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) - disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') + disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o', + policy=POLICIES.legacy) disk_file.open() file_name = os.path.basename(disk_file._data_file) @@ -1133,7 +1417,7 @@ class TestObjectController(unittest.TestCase): resp = server_handler.OPTIONS(req) self.assertEquals(200, resp.status_int) for verb in 'OPTIONS GET POST PUT DELETE HEAD REPLICATE \ - REPLICATION'.split(): + SSYNC'.split(): self.assertTrue( verb in resp.headers['Allow'].split(', ')) self.assertEquals(len(resp.headers['Allow'].split(', ')), 8) @@ -1201,7 +1485,7 @@ class TestObjectController(unittest.TestCase): objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.data') os.unlink(objfile) @@ -1290,6 +1574,58 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) + def test_GET_if_match_etag_is_at(self): + headers = { + 'X-Timestamp': utils.Timestamp(time()).internal, + 'Content-Type': 'application/octet-stream', + 'X-Object-Meta-Xtag': 'madeup', + } + req = Request.blank('/sda1/p/a/c/o', method='PUT', + headers=headers) + req.body = 'test' + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 201) + real_etag = resp.etag + + # match x-backend-etag-is-at + req = Request.blank('/sda1/p/a/c/o', headers={ + 'If-Match': 'madeup', + 'X-Backend-Etag-Is-At': 'X-Object-Meta-Xtag'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + + # no match x-backend-etag-is-at + req = Request.blank('/sda1/p/a/c/o', headers={ + 'If-Match': real_etag, + 'X-Backend-Etag-Is-At': 'X-Object-Meta-Xtag'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 412) + + # etag-is-at metadata doesn't exist, default to real etag + req = Request.blank('/sda1/p/a/c/o', headers={ + 'If-Match': real_etag, + 'X-Backend-Etag-Is-At': 'X-Object-Meta-Missing'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + + # sanity no-match with no etag-is-at + req = Request.blank('/sda1/p/a/c/o', headers={ + 'If-Match': 'madeup'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 412) + + # sanity match with no etag-is-at + req = Request.blank('/sda1/p/a/c/o', headers={ + 'If-Match': real_etag}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + + # sanity with no if-match + req = Request.blank('/sda1/p/a/c/o', headers={ + 'X-Backend-Etag-Is-At': 'X-Object-Meta-Xtag'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + def test_HEAD_if_match(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ @@ -1692,7 +2028,8 @@ class TestObjectController(unittest.TestCase): req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) - disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') + disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o', + policy=POLICIES.legacy) disk_file.open() file_name = os.path.basename(disk_file._data_file) etag = md5() @@ -1724,7 +2061,8 @@ class TestObjectController(unittest.TestCase): req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) - disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') + disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o', + policy=POLICIES.legacy) disk_file.open() file_name = os.path.basename(disk_file._data_file) with open(disk_file._data_file) as fp: @@ -1752,7 +2090,8 @@ class TestObjectController(unittest.TestCase): req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) - disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') + disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o', + policy=POLICIES.legacy) disk_file.open() file_name = os.path.basename(disk_file._data_file) etag = md5() @@ -1810,7 +2149,6 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'DELETE'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) - # self.assertRaises(KeyError, self.object_controller.DELETE, req) # The following should have created a tombstone file timestamp = normalize_timestamp(1000) @@ -1821,7 +2159,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 404) ts_1000_file = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assertTrue(os.path.isfile(ts_1000_file)) @@ -1837,7 +2175,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 404) ts_999_file = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(ts_999_file)) @@ -1857,7 +2195,7 @@ class TestObjectController(unittest.TestCase): # There should now be 1000 ts and a 1001 data file. data_1002_file = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), orig_timestamp + '.data') self.assertTrue(os.path.isfile(data_1002_file)) @@ -1873,7 +2211,7 @@ class TestObjectController(unittest.TestCase): self.assertEqual(resp.headers['X-Backend-Timestamp'], orig_timestamp) ts_1001_file = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(ts_1001_file)) @@ -1888,7 +2226,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 204) ts_1003_file = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assertTrue(os.path.isfile(ts_1003_file)) @@ -1930,7 +2268,7 @@ class TestObjectController(unittest.TestCase): orig_timestamp.internal) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(objfile)) @@ -1949,7 +2287,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 204) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assert_(os.path.isfile(objfile)) @@ -1968,7 +2306,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assert_(os.path.isfile(objfile)) @@ -1987,7 +2325,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(objfile)) @@ -2184,7 +2522,7 @@ class TestObjectController(unittest.TestCase): def test_call_bad_request(self): # Test swift.obj.server.ObjectController.__call__ - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() @@ -2211,7 +2549,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(outbuf.getvalue()[:4], '400 ') def test_call_not_found(self): - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() @@ -2238,7 +2576,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(outbuf.getvalue()[:4], '404 ') def test_call_bad_method(self): - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() @@ -2274,7 +2612,7 @@ class TestObjectController(unittest.TestCase): with mock.patch("swift.obj.diskfile.hash_path", my_hash_path): with mock.patch("swift.obj.server.check_object_creation", my_check): - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() @@ -2303,7 +2641,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '201 ') - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() @@ -2454,6 +2792,9 @@ class TestObjectController(unittest.TestCase): return ' ' return '' + def set_hundred_continue_response_headers(*a, **kw): + pass + req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, @@ -2483,6 +2824,9 @@ class TestObjectController(unittest.TestCase): return ' ' return '' + def set_hundred_continue_response_headers(*a, **kw): + pass + req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': ShortBody()}, @@ -2554,8 +2898,8 @@ class TestObjectController(unittest.TestCase): self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': 'set', - 'X-Backend-Storage-Policy-Index': policy.idx}, 'sda1', - policy.idx) + 'X-Backend-Storage-Policy-Index': int(policy)}, 'sda1', + policy) finally: object_server.http_connect = orig_http_connect self.assertEquals( @@ -2563,12 +2907,15 @@ class TestObjectController(unittest.TestCase): ['127.0.0.1', '1234', 'sdc1', 1, 'PUT', '/a/c/o', { 'x-timestamp': '1', 'x-out': 'set', 'user-agent': 'object-server %s' % os.getpid(), - 'X-Backend-Storage-Policy-Index': policy.idx}]) + 'X-Backend-Storage-Policy-Index': int(policy)}]) - @patch_policies([storage_policy.StoragePolicy(0, 'zero', True), - storage_policy.StoragePolicy(1, 'one'), - storage_policy.StoragePolicy(37, 'fantastico')]) + @patch_policies([StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one'), + StoragePolicy(37, 'fantastico')]) def test_updating_multiple_delete_at_container_servers(self): + # update router post patch + self.object_controller._diskfile_router = diskfile.DiskFileRouter( + self.conf, self.object_controller.logger) policy = random.choice(list(POLICIES)) self.object_controller.expiring_objects_account = 'exp' self.object_controller.expiring_objects_container_divisor = 60 @@ -2607,7 +2954,7 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': '12345', 'Content-Type': 'application/burrito', 'Content-Length': '0', - 'X-Backend-Storage-Policy-Index': policy.idx, + 'X-Backend-Storage-Policy-Index': int(policy), 'X-Container-Partition': '20', 'X-Container-Host': '1.2.3.4:5', 'X-Container-Device': 'sdb1', @@ -2643,7 +2990,7 @@ class TestObjectController(unittest.TestCase): 'X-Backend-Storage-Policy-Index': '37', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'object-server %d' % os.getpid(), - 'X-Backend-Storage-Policy-Index': policy.idx, + 'X-Backend-Storage-Policy-Index': int(policy), 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[1], @@ -2684,10 +3031,13 @@ class TestObjectController(unittest.TestCase): 'X-Backend-Storage-Policy-Index': 0, 'x-trans-id': '-'})}) - @patch_policies([storage_policy.StoragePolicy(0, 'zero', True), - storage_policy.StoragePolicy(1, 'one'), - storage_policy.StoragePolicy(26, 'twice-thirteen')]) + @patch_policies([StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one'), + StoragePolicy(26, 'twice-thirteen')]) def test_updating_multiple_container_servers(self): + # update router post patch + self.object_controller._diskfile_router = diskfile.DiskFileRouter( + self.conf, self.object_controller.logger) http_connect_args = [] def fake_http_connect(ipaddr, port, device, partition, method, path, @@ -2788,7 +3138,7 @@ class TestObjectController(unittest.TestCase): int(delete_at_timestamp) / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) - req = Request.blank('/sda1/p/a/c/o', method='PUT', body='', headers={ + headers = { 'Content-Type': 'text/plain', 'X-Timestamp': put_timestamp, 'X-Container-Host': '10.0.0.1:6001', @@ -2799,8 +3149,11 @@ class TestObjectController(unittest.TestCase): 'X-Delete-At-Partition': 'p', 'X-Delete-At-Host': '10.0.0.2:6002', 'X-Delete-At-Device': 'sda1', - 'X-Backend-Storage-Policy-Index': int(policy), - }) + 'X-Backend-Storage-Policy-Index': int(policy)} + if policy.policy_type == EC_POLICY: + headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2' + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', body='', headers=headers) with mocked_http_conn( 500, 500, give_connect=capture_updates) as fake_conn: resp = req.get_response(self.object_controller) @@ -2836,7 +3189,7 @@ class TestObjectController(unittest.TestCase): self.assertEqual(headers[key], str(value)) # check async pendings async_dir = os.path.join(self.testdir, 'sda1', - diskfile.get_async_dir(policy.idx)) + diskfile.get_async_dir(policy)) found_files = [] for root, dirs, files in os.walk(async_dir): for f in files: @@ -2846,7 +3199,7 @@ class TestObjectController(unittest.TestCase): if data['account'] == 'a': self.assertEquals( int(data['headers'] - ['X-Backend-Storage-Policy-Index']), policy.idx) + ['X-Backend-Storage-Policy-Index']), int(policy)) elif data['account'] == '.expiring_objects': self.assertEquals( int(data['headers'] @@ -2870,12 +3223,12 @@ class TestObjectController(unittest.TestCase): self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': 'set', - 'X-Backend-Storage-Policy-Index': policy.idx}, 'sda1', - policy.idx) + 'X-Backend-Storage-Policy-Index': int(policy)}, 'sda1', + policy) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix - async_dir = diskfile.get_async_dir(policy.idx) + async_dir = diskfile.get_async_dir(policy) self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', @@ -2883,7 +3236,7 @@ class TestObjectController(unittest.TestCase): utils.Timestamp(1).internal))), {'headers': {'x-timestamp': '1', 'x-out': 'set', 'user-agent': 'object-server %s' % os.getpid(), - 'X-Backend-Storage-Policy-Index': policy.idx}, + 'X-Backend-Storage-Policy-Index': int(policy)}, 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) def test_async_update_saves_on_non_2xx(self): @@ -2914,9 +3267,9 @@ class TestObjectController(unittest.TestCase): self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': str(status), - 'X-Backend-Storage-Policy-Index': policy.idx}, 'sda1', - policy.idx) - async_dir = diskfile.get_async_dir(policy.idx) + 'X-Backend-Storage-Policy-Index': int(policy)}, 'sda1', + policy) + async_dir = diskfile.get_async_dir(policy) self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', @@ -2926,7 +3279,7 @@ class TestObjectController(unittest.TestCase): 'user-agent': 'object-server %s' % os.getpid(), 'X-Backend-Storage-Policy-Index': - policy.idx}, + int(policy)}, 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) finally: @@ -2990,8 +3343,8 @@ class TestObjectController(unittest.TestCase): self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': str(status)}, 'sda1', - policy.idx) - async_dir = diskfile.get_async_dir(int(policy)) + policy) + async_dir = diskfile.get_async_dir(policy) self.assertTrue( os.path.exists(os.path.join( self.testdir, 'sda1', async_dir, 'a83', @@ -3002,6 +3355,7 @@ class TestObjectController(unittest.TestCase): utils.HASH_PATH_PREFIX = _prefix def test_container_update_no_async_update(self): + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3012,12 +3366,13 @@ class TestObjectController(unittest.TestCase): '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, - 'X-Trans-Id': '1234'}) + 'X-Trans-Id': '1234', + 'X-Backend-Storage-Policy-Index': int(policy)}) self.object_controller.container_update( 'PUT', 'a', 'c', 'o', req, { 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1'}, - 'sda1', 0) + 'sda1', policy) self.assertEquals(given_args, []) def test_container_update_success(self): @@ -3099,6 +3454,7 @@ class TestObjectController(unittest.TestCase): 'x-foo': 'bar'})) def test_container_update_async(self): + policy = random.choice(list(POLICIES)) req = Request.blank( '/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -3107,26 +3463,28 @@ class TestObjectController(unittest.TestCase): 'X-Container-Host': 'chost:cport', 'X-Container-Partition': 'cpartition', 'X-Container-Device': 'cdevice', - 'Content-Type': 'text/plain'}, body='') + 'Content-Type': 'text/plain', + 'X-Object-Sysmeta-Ec-Frag-Index': 0, + 'X-Backend-Storage-Policy-Index': int(policy)}, body='') given_args = [] def fake_pickle_async_update(*args): given_args[:] = args - self.object_controller._diskfile_mgr.pickle_async_update = \ - fake_pickle_async_update + diskfile_mgr = self.object_controller._diskfile_router[policy] + diskfile_mgr.pickle_async_update = fake_pickle_async_update with mocked_http_conn(500) as fake_conn: resp = req.get_response(self.object_controller) self.assertRaises(StopIteration, fake_conn.code_iter.next) self.assertEqual(resp.status_int, 201) self.assertEqual(len(given_args), 7) (objdevice, account, container, obj, data, timestamp, - policy_index) = given_args + policy) = given_args self.assertEqual(objdevice, 'sda1') self.assertEqual(account, 'a') self.assertEqual(container, 'c') self.assertEqual(obj, 'o') self.assertEqual(timestamp, utils.Timestamp(1).internal) - self.assertEqual(policy_index, 0) + self.assertEqual(policy, policy) self.assertEqual(data, { 'headers': HeaderKeyDict({ 'X-Size': '0', @@ -3135,7 +3493,7 @@ class TestObjectController(unittest.TestCase): 'X-Timestamp': utils.Timestamp(1).internal, 'X-Trans-Id': '123', 'Referer': 'PUT http://localhost/sda1/0/a/c/o', - 'X-Backend-Storage-Policy-Index': '0', + 'X-Backend-Storage-Policy-Index': int(policy), 'X-Etag': 'd41d8cd98f00b204e9800998ecf8427e'}), 'obj': 'o', 'account': 'a', @@ -3143,6 +3501,7 @@ class TestObjectController(unittest.TestCase): 'op': 'PUT'}) def test_container_update_bad_args(self): + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3155,7 +3514,8 @@ class TestObjectController(unittest.TestCase): 'X-Trans-Id': '123', 'X-Container-Host': 'chost,badhost', 'X-Container-Partition': 'cpartition', - 'X-Container-Device': 'cdevice'}) + 'X-Container-Device': 'cdevice', + 'X-Backend-Storage-Policy-Index': int(policy)}) with mock.patch.object(self.object_controller, 'async_update', fake_async_update): self.object_controller.container_update( @@ -3163,7 +3523,7 @@ class TestObjectController(unittest.TestCase): 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1'}, - 'sda1', 0) + 'sda1', policy) self.assertEqual(given_args, []) errors = self.object_controller.logger.get_lines_for_level('error') self.assertEqual(len(errors), 1) @@ -3176,6 +3536,7 @@ class TestObjectController(unittest.TestCase): def test_delete_at_update_on_put(self): # Test how delete_at_update works when issued a delete for old # expiration info after a new put with no new expiration info. + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3185,11 +3546,12 @@ class TestObjectController(unittest.TestCase): '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, - 'X-Trans-Id': '123'}) + 'X-Trans-Id': '123', + 'X-Backend-Storage-Policy-Index': int(policy)}) with mock.patch.object(self.object_controller, 'async_update', fake_async_update): self.object_controller.delete_at_update( - 'DELETE', 2, 'a', 'c', 'o', req, 'sda1', 0) + 'DELETE', 2, 'a', 'c', 'o', req, 'sda1', policy) self.assertEquals( given_args, [ 'DELETE', '.expiring_objects', '0000000000', @@ -3199,12 +3561,13 @@ class TestObjectController(unittest.TestCase): 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}), - 'sda1', 0]) + 'sda1', policy]) def test_delete_at_negative(self): # Test how delete_at_update works when issued a delete for old # expiration info after a new put with no new expiration info. # Test negative is reset to 0 + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3215,23 +3578,26 @@ class TestObjectController(unittest.TestCase): '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, - 'X-Trans-Id': '1234'}) + 'X-Trans-Id': '1234', 'X-Backend-Storage-Policy-Index': + int(policy)}) self.object_controller.delete_at_update( - 'DELETE', -2, 'a', 'c', 'o', req, 'sda1', 0) + 'DELETE', -2, 'a', 'c', 'o', req, 'sda1', policy) self.assertEquals(given_args, [ 'DELETE', '.expiring_objects', '0000000000', '0000000000-a/c/o', None, None, None, HeaderKeyDict({ + # the expiring objects account is always 0 'X-Backend-Storage-Policy-Index': 0, 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), - 'sda1', 0]) + 'sda1', policy]) def test_delete_at_cap(self): # Test how delete_at_update works when issued a delete for old # expiration info after a new put with no new expiration info. # Test past cap is reset to cap + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3242,9 +3608,10 @@ class TestObjectController(unittest.TestCase): '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, - 'X-Trans-Id': '1234'}) + 'X-Trans-Id': '1234', + 'X-Backend-Storage-Policy-Index': int(policy)}) self.object_controller.delete_at_update( - 'DELETE', 12345678901, 'a', 'c', 'o', req, 'sda1', 0) + 'DELETE', 12345678901, 'a', 'c', 'o', req, 'sda1', policy) expiring_obj_container = given_args.pop(2) expected_exp_cont = utils.get_expirer_container( utils.normalize_delete_at_timestamp(12345678901), @@ -3259,12 +3626,13 @@ class TestObjectController(unittest.TestCase): 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), - 'sda1', 0]) + 'sda1', policy]) def test_delete_at_update_put_with_info(self): # Keep next test, # test_delete_at_update_put_with_info_but_missing_container, in sync # with this one but just missing the X-Delete-At-Container header. + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3279,14 +3647,16 @@ class TestObjectController(unittest.TestCase): 'X-Delete-At-Container': '0', 'X-Delete-At-Host': '127.0.0.1:1234', 'X-Delete-At-Partition': '3', - 'X-Delete-At-Device': 'sdc1'}) + 'X-Delete-At-Device': 'sdc1', + 'X-Backend-Storage-Policy-Index': int(policy)}) self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', - req, 'sda1', 0) + req, 'sda1', policy) self.assertEquals( given_args, [ 'PUT', '.expiring_objects', '0000000000', '0000000002-a/c/o', '127.0.0.1:1234', '3', 'sdc1', HeaderKeyDict({ + # the .expiring_objects account is always policy-0 'X-Backend-Storage-Policy-Index': 0, 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', @@ -3294,18 +3664,19 @@ class TestObjectController(unittest.TestCase): 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), - 'sda1', 0]) + 'sda1', policy]) def test_delete_at_update_put_with_info_but_missing_container(self): # Same as previous test, test_delete_at_update_put_with_info, but just # missing the X-Delete-At-Container header. + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update - self.object_controller.logger = FakeLogger() + self.object_controller.logger = self.logger req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -3313,16 +3684,18 @@ class TestObjectController(unittest.TestCase): 'X-Trans-Id': '1234', 'X-Delete-At-Host': '127.0.0.1:1234', 'X-Delete-At-Partition': '3', - 'X-Delete-At-Device': 'sdc1'}) + 'X-Delete-At-Device': 'sdc1', + 'X-Backend-Storage-Policy-Index': int(policy)}) self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', - req, 'sda1', 0) + req, 'sda1', policy) self.assertEquals( - self.object_controller.logger.log_dict['warning'], - [(('X-Delete-At-Container header must be specified for expiring ' - 'objects background PUT to work properly. Making best guess as ' - 'to the container name for now.',), {})]) + self.logger.get_lines_for_level('warning'), + ['X-Delete-At-Container header must be specified for expiring ' + 'objects background PUT to work properly. Making best guess as ' + 'to the container name for now.']) def test_delete_at_update_delete(self): + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3333,9 +3706,10 @@ class TestObjectController(unittest.TestCase): '/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': 1, - 'X-Trans-Id': '1234'}) + 'X-Trans-Id': '1234', + 'X-Backend-Storage-Policy-Index': int(policy)}) self.object_controller.delete_at_update('DELETE', 2, 'a', 'c', 'o', - req, 'sda1', 0) + req, 'sda1', policy) self.assertEquals( given_args, [ 'DELETE', '.expiring_objects', '0000000000', @@ -3345,11 +3719,12 @@ class TestObjectController(unittest.TestCase): 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'DELETE http://localhost/v1/a/c/o'}), - 'sda1', 0]) + 'sda1', policy]) def test_delete_backend_replication(self): # If X-Backend-Replication: True delete_at_update should completely # short-circuit. + policy = random.choice(list(POLICIES)) given_args = [] def fake_async_update(*args): @@ -3361,12 +3736,14 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234', - 'X-Backend-Replication': 'True'}) + 'X-Backend-Replication': 'True', + 'X-Backend-Storage-Policy-Index': int(policy)}) self.object_controller.delete_at_update( - 'DELETE', -2, 'a', 'c', 'o', req, 'sda1', 0) + 'DELETE', -2, 'a', 'c', 'o', req, 'sda1', policy) self.assertEquals(given_args, []) def test_POST_calls_delete_at(self): + policy = random.choice(list(POLICIES)) given_args = [] def fake_delete_at_update(*args): @@ -3378,7 +3755,9 @@ class TestObjectController(unittest.TestCase): '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', - 'Content-Type': 'application/octet-stream'}) + 'Content-Type': 'application/octet-stream', + 'X-Backend-Storage-Policy-Index': int(policy), + 'X-Object-Sysmeta-Ec-Frag-Index': 2}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) @@ -3389,7 +3768,8 @@ class TestObjectController(unittest.TestCase): '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(time()), - 'Content-Type': 'application/x-test'}) + 'Content-Type': 'application/x-test', + 'X-Backend-Storage-Policy-Index': int(policy)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) self.assertEquals(given_args, []) @@ -3402,13 +3782,14 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp1, 'Content-Type': 'application/x-test', - 'X-Delete-At': delete_at_timestamp1}) + 'X-Delete-At': delete_at_timestamp1, + 'X-Backend-Storage-Policy-Index': int(policy)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', - given_args[5], 'sda1', 0]) + given_args[5], 'sda1', policy]) while given_args: given_args.pop() @@ -3421,17 +3802,19 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp2, 'Content-Type': 'application/x-test', - 'X-Delete-At': delete_at_timestamp2}) + 'X-Delete-At': delete_at_timestamp2, + 'X-Backend-Storage-Policy-Index': int(policy)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp2), 'a', 'c', 'o', - given_args[5], 'sda1', 0, + given_args[5], 'sda1', policy, 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', - given_args[5], 'sda1', 0]) + given_args[5], 'sda1', policy]) def test_PUT_calls_delete_at(self): + policy = random.choice(list(POLICIES)) given_args = [] def fake_delete_at_update(*args): @@ -3443,7 +3826,9 @@ class TestObjectController(unittest.TestCase): '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', - 'Content-Type': 'application/octet-stream'}) + 'Content-Type': 'application/octet-stream', + 'X-Backend-Storage-Policy-Index': int(policy), + 'X-Object-Sysmeta-Ec-Frag-Index': 4}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) @@ -3457,14 +3842,16 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': timestamp1, 'Content-Length': '4', 'Content-Type': 'application/octet-stream', - 'X-Delete-At': delete_at_timestamp1}) + 'X-Delete-At': delete_at_timestamp1, + 'X-Backend-Storage-Policy-Index': int(policy), + 'X-Object-Sysmeta-Ec-Frag-Index': 3}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', - given_args[5], 'sda1', 0]) + given_args[5], 'sda1', policy]) while given_args: given_args.pop() @@ -3478,16 +3865,18 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': timestamp2, 'Content-Length': '4', 'Content-Type': 'application/octet-stream', - 'X-Delete-At': delete_at_timestamp2}) + 'X-Delete-At': delete_at_timestamp2, + 'X-Backend-Storage-Policy-Index': int(policy), + 'X-Object-Sysmeta-Ec-Frag-Index': 3}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp2), 'a', 'c', 'o', - given_args[5], 'sda1', 0, + given_args[5], 'sda1', policy, 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', - given_args[5], 'sda1', 0]) + given_args[5], 'sda1', policy]) def test_GET_but_expired(self): test_time = time() + 10000 @@ -3742,7 +4131,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.body, 'TEST') objfile = os.path.join( self.testdir, 'sda1', - storage_directory(diskfile.get_data_dir(0), 'p', + storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', hash_path('a', 'c', 'o')), utils.Timestamp(test_timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) @@ -3909,7 +4298,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', - given_args[5], 'sda1', 0]) + given_args[5], 'sda1', POLICIES[0]]) while given_args: given_args.pop() @@ -3925,7 +4314,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assertEquals(given_args, [ 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', - given_args[5], 'sda1', 0]) + given_args[5], 'sda1', POLICIES[0]]) def test_PUT_delete_at_in_past(self): req = Request.blank( @@ -3967,10 +4356,10 @@ class TestObjectController(unittest.TestCase): def my_tpool_execute(func, *args, **kwargs): return func(*args, **kwargs) - was_get_hashes = diskfile.get_hashes + was_get_hashes = diskfile.DiskFileManager._get_hashes was_tpool_exe = tpool.execute try: - diskfile.get_hashes = fake_get_hashes + diskfile.DiskFileManager._get_hashes = fake_get_hashes tpool.execute = my_tpool_execute req = Request.blank('/sda1/p/suff', environ={'REQUEST_METHOD': 'REPLICATE'}, @@ -3981,7 +4370,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(p_data, {1: 2}) finally: tpool.execute = was_tpool_exe - diskfile.get_hashes = was_get_hashes + diskfile.DiskFileManager._get_hashes = was_get_hashes def test_REPLICATE_timeout(self): @@ -3991,10 +4380,10 @@ class TestObjectController(unittest.TestCase): def my_tpool_execute(func, *args, **kwargs): return func(*args, **kwargs) - was_get_hashes = diskfile.get_hashes + was_get_hashes = diskfile.DiskFileManager._get_hashes was_tpool_exe = tpool.execute try: - diskfile.get_hashes = fake_get_hashes + diskfile.DiskFileManager._get_hashes = fake_get_hashes tpool.execute = my_tpool_execute req = Request.blank('/sda1/p/suff', environ={'REQUEST_METHOD': 'REPLICATE'}, @@ -4002,7 +4391,7 @@ class TestObjectController(unittest.TestCase): self.assertRaises(Timeout, self.object_controller.REPLICATE, req) finally: tpool.execute = was_tpool_exe - diskfile.get_hashes = was_get_hashes + diskfile.DiskFileManager._get_hashes = was_get_hashes def test_REPLICATE_insufficient_storage(self): conf = {'devices': self.testdir, 'mount_check': 'true'} @@ -4020,9 +4409,9 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 507) - def test_REPLICATION_can_be_called(self): + def test_SSYNC_can_be_called(self): req = Request.blank('/sda1/p/other/suff', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, headers={}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) @@ -4113,7 +4502,7 @@ class TestObjectController(unittest.TestCase): def test_list_allowed_methods(self): # Test list of allowed_methods obj_methods = ['DELETE', 'PUT', 'HEAD', 'GET', 'POST'] - repl_methods = ['REPLICATE', 'REPLICATION'] + repl_methods = ['REPLICATE', 'SSYNC'] for method_name in obj_methods: method = getattr(self.object_controller, method_name) self.assertFalse(hasattr(method, 'replication')) @@ -4124,7 +4513,7 @@ class TestObjectController(unittest.TestCase): def test_correct_allowed_method(self): # Test correct work for allowed method using # swift.obj.server.ObjectController.__call__ - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.app_factory( @@ -4162,12 +4551,12 @@ class TestObjectController(unittest.TestCase): def test_not_allowed_method(self): # Test correct work for NOT allowed method using # swift.obj.server.ObjectController.__call__ - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.ObjectController( {'devices': self.testdir, 'mount_check': 'false', - 'replication_server': 'false'}, logger=FakeLogger()) + 'replication_server': 'false'}, logger=self.logger) def start_response(*args): # Sends args to outbuf @@ -4207,11 +4596,10 @@ class TestObjectController(unittest.TestCase): env, start_response) self.assertEqual(response, answer) self.assertEqual( - self.object_controller.logger.log_dict['info'], - [(('None - - [01/Jan/1970:02:46:41 +0000] "PUT' - ' /sda1/p/a/c/o" 405 - "-" "-" "-" 1.0000 "-"' - ' 1234 -',), - {})]) + self.logger.get_lines_for_level('info'), + ['None - - [01/Jan/1970:02:46:41 +0000] "PUT' + ' /sda1/p/a/c/o" 405 - "-" "-" "-" 1.0000 "-"' + ' 1234 -']) def test_call_incorrect_replication_method(self): inbuf = StringIO() @@ -4246,7 +4634,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(outbuf.getvalue()[:4], '405 ') def test_not_utf8_and_not_logging_requests(self): - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.ObjectController( @@ -4281,17 +4669,17 @@ class TestObjectController(unittest.TestCase): new=mock_method): response = self.object_controller.__call__(env, start_response) self.assertEqual(response, answer) - self.assertEqual(self.object_controller.logger.log_dict['info'], - []) + self.assertEqual(self.logger.get_lines_for_level('info'), []) def test__call__returns_500(self): - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() + self.logger = debug_logger('test') self.object_controller = object_server.ObjectController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false', 'log_requests': 'false'}, - logger=FakeLogger()) + logger=self.logger) def start_response(*args): # Sends args to outbuf @@ -4323,24 +4711,21 @@ class TestObjectController(unittest.TestCase): response = self.object_controller.__call__(env, start_response) self.assertTrue(response[0].startswith( 'Traceback (most recent call last):')) - self.assertEqual( - self.object_controller.logger.log_dict['exception'], - [(('ERROR __call__ error with %(method)s %(path)s ', - {'method': 'PUT', 'path': '/sda1/p/a/c/o'}), - {}, - '')]) - self.assertEqual(self.object_controller.logger.log_dict['INFO'], - []) + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'ERROR __call__ error with %(method)s %(path)s : ' % { + 'method': 'PUT', 'path': '/sda1/p/a/c/o'}, + ]) + self.assertEqual(self.logger.get_lines_for_level('info'), []) def test_PUT_slow(self): - inbuf = StringIO() + inbuf = WsgiStringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.ObjectController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false', 'log_requests': 'false', 'slow': '10'}, - logger=FakeLogger()) + logger=self.logger) def start_response(*args): # Sends args to outbuf @@ -4373,14 +4758,14 @@ class TestObjectController(unittest.TestCase): mock.MagicMock()) as ms: self.object_controller.__call__(env, start_response) ms.assert_called_with(9) - self.assertEqual( - self.object_controller.logger.log_dict['info'], []) + self.assertEqual(self.logger.get_lines_for_level('info'), + []) def test_log_line_format(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'}) - self.object_controller.logger = FakeLogger() + self.object_controller.logger = self.logger with mock.patch( 'time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])): with mock.patch( @@ -4390,13 +4775,16 @@ class TestObjectController(unittest.TestCase): 'os.getpid', mock.MagicMock(return_value=1234)): req.get_response(self.object_controller) self.assertEqual( - self.object_controller.logger.log_dict['info'], - [(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a/c/o" ' - '404 - "-" "-" "-" 2.0000 "-" 1234 -',), {})]) + self.logger.get_lines_for_level('info'), + ['1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a/c/o" ' + '404 - "-" "-" "-" 2.0000 "-" 1234 -']) - @patch_policies([storage_policy.StoragePolicy(0, 'zero', True), - storage_policy.StoragePolicy(1, 'one', False)]) + @patch_policies([StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one', False)]) def test_dynamic_datadir(self): + # update router post patch + self.object_controller._diskfile_router = diskfile.DiskFileRouter( + self.conf, self.object_controller.logger) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, @@ -4430,7 +4818,50 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertTrue(os.path.isdir(object_dir)) + def test_storage_policy_index_is_validated(self): + # sanity check that index for existing policy is ok + ts = (utils.Timestamp(t).internal for t in + itertools.count(int(time()))) + methods = ('PUT', 'POST', 'GET', 'HEAD', 'REPLICATE', 'DELETE') + valid_indices = sorted([int(policy) for policy in POLICIES]) + for index in valid_indices: + object_dir = self.testdir + "/sda1/objects" + if index > 0: + object_dir = "%s-%s" % (object_dir, index) + self.assertFalse(os.path.isdir(object_dir)) + for method in methods: + headers = { + 'X-Timestamp': ts.next(), + 'Content-Type': 'application/x-test', + 'X-Backend-Storage-Policy-Index': index} + if POLICIES[index].policy_type == EC_POLICY: + headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2' + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': method}, + headers=headers) + req.body = 'VERIFY' + resp = req.get_response(self.object_controller) + self.assertTrue(is_success(resp.status_int), + '%s method failed: %r' % (method, resp.status)) + + # index for non-existent policy should return 503 + index = valid_indices[-1] + 1 + for method in methods: + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': method}, + headers={ + 'X-Timestamp': ts.next(), + 'Content-Type': 'application/x-test', + 'X-Backend-Storage-Policy-Index': index}) + req.body = 'VERIFY' + object_dir = self.testdir + "/sda1/objects-%s" % index + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 503) + self.assertFalse(os.path.isdir(object_dir)) + +@patch_policies(test_policies) class TestObjectServer(unittest.TestCase): def setUp(self): @@ -4442,13 +4873,13 @@ class TestObjectServer(unittest.TestCase): for device in ('sda1', 'sdb1'): os.makedirs(os.path.join(self.devices, device)) - conf = { + self.conf = { 'devices': self.devices, 'swift_dir': self.tempdir, 'mount_check': 'false', } self.logger = debug_logger('test-object-server') - app = object_server.ObjectController(conf, logger=self.logger) + app = object_server.ObjectController(self.conf, logger=self.logger) sock = listen(('127.0.0.1', 0)) self.server = spawn(wsgi.server, sock, app, utils.NullLogger()) self.port = sock.getsockname()[1] @@ -4481,6 +4912,23 @@ class TestObjectServer(unittest.TestCase): resp.read() resp.close() + def test_expect_on_put_footer(self): + test_body = 'test' + headers = { + 'Expect': '100-continue', + 'Content-Length': len(test_body), + 'X-Timestamp': utils.Timestamp(time()).internal, + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary123', + } + conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0', + 'PUT', '/a/c/o', headers=headers) + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + headers = HeaderKeyDict(resp.getheaders()) + self.assertEqual(headers['X-Obj-Metadata-Footer'], 'yes') + resp.close() + def test_expect_on_put_conflict(self): test_body = 'test' put_timestamp = utils.Timestamp(time()) @@ -4509,7 +4957,379 @@ class TestObjectServer(unittest.TestCase): resp.read() resp.close() + def test_multiphase_put_no_mime_boundary(self): + test_data = 'obj data' + put_timestamp = utils.Timestamp(time()).internal + headers = { + 'Content-Type': 'text/plain', + 'X-Timestamp': put_timestamp, + 'Transfer-Encoding': 'chunked', + 'Expect': '100-continue', + 'X-Backend-Obj-Content-Length': len(test_data), + 'X-Backend-Obj-Multiphase-Commit': 'yes', + } + conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0', + 'PUT', '/a/c/o', headers=headers) + resp = conn.getexpect() + self.assertEqual(resp.status, 400) + resp.read() + resp.close() + + def test_expect_on_multiphase_put(self): + test_data = 'obj data' + test_doc = "\r\n".join(( + "--boundary123", + "X-Document: object body", + "", + test_data, + "--boundary123", + )) + + put_timestamp = utils.Timestamp(time()).internal + headers = { + 'Content-Type': 'text/plain', + 'X-Timestamp': put_timestamp, + 'Transfer-Encoding': 'chunked', + 'Expect': '100-continue', + 'X-Backend-Obj-Content-Length': len(test_data), + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary123', + 'X-Backend-Obj-Multiphase-Commit': 'yes', + } + conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0', + 'PUT', '/a/c/o', headers=headers) + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + headers = HeaderKeyDict(resp.getheaders()) + self.assertEqual(headers['X-Obj-Multiphase-Commit'], 'yes') + + to_send = "%x\r\n%s\r\n0\r\n\r\n" % (len(test_doc), test_doc) + conn.send(to_send) + + # verify 100-continue response to mark end of phase1 + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + resp.close() + + def test_multiphase_put_metadata_footer(self): + # Test 2-phase commit conversation - end of 1st phase marked + # by 100-continue response from the object server, with a + # successful 2nd phase marked by the presence of a .durable + # file along with .data file in the object data directory + test_data = 'obj data' + footer_meta = { + "X-Object-Sysmeta-Ec-Frag-Index": "2", + "Etag": md5(test_data).hexdigest(), + } + footer_json = json.dumps(footer_meta) + footer_meta_cksum = md5(footer_json).hexdigest() + test_doc = "\r\n".join(( + "--boundary123", + "X-Document: object body", + "", + test_data, + "--boundary123", + "X-Document: object metadata", + "Content-MD5: " + footer_meta_cksum, + "", + footer_json, + "--boundary123", + )) + + # phase1 - PUT request with object metadata in footer and + # multiphase commit conversation + put_timestamp = utils.Timestamp(time()).internal + headers = { + 'Content-Type': 'text/plain', + 'X-Timestamp': put_timestamp, + 'Transfer-Encoding': 'chunked', + 'Expect': '100-continue', + 'X-Backend-Storage-Policy-Index': '1', + 'X-Backend-Obj-Content-Length': len(test_data), + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary123', + 'X-Backend-Obj-Multiphase-Commit': 'yes', + } + conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0', + 'PUT', '/a/c/o', headers=headers) + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + headers = HeaderKeyDict(resp.getheaders()) + self.assertEqual(headers['X-Obj-Multiphase-Commit'], 'yes') + self.assertEqual(headers['X-Obj-Metadata-Footer'], 'yes') + + to_send = "%x\r\n%s\r\n0\r\n\r\n" % (len(test_doc), test_doc) + conn.send(to_send) + # verify 100-continue response to mark end of phase1 + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + + # send commit confirmation to start phase2 + commit_confirmation_doc = "\r\n".join(( + "X-Document: put commit", + "", + "commit_confirmation", + "--boundary123--", + )) + to_send = "%x\r\n%s\r\n0\r\n\r\n" % \ + (len(commit_confirmation_doc), commit_confirmation_doc) + conn.send(to_send) + + # verify success (2xx) to make end of phase2 + resp = conn.getresponse() + self.assertEqual(resp.status, 201) + resp.read() + resp.close() + + # verify successful object data and durable state file write + obj_basename = os.path.join( + self.devices, 'sda1', + storage_directory(diskfile.get_data_dir(POLICIES[1]), '0', + hash_path('a', 'c', 'o')), + put_timestamp) + obj_datafile = obj_basename + '#2.data' + self.assertTrue(os.path.isfile(obj_datafile)) + obj_durablefile = obj_basename + '.durable' + self.assertTrue(os.path.isfile(obj_durablefile)) + + def test_multiphase_put_no_metadata_footer(self): + # Test 2-phase commit conversation, with no metadata footer + # at the end of object data - end of 1st phase marked + # by 100-continue response from the object server, with a + # successful 2nd phase marked by the presence of a .durable + # file along with .data file in the object data directory + # (No metadata footer case) + test_data = 'obj data' + test_doc = "\r\n".join(( + "--boundary123", + "X-Document: object body", + "", + test_data, + "--boundary123", + )) + + # phase1 - PUT request with multiphase commit conversation + # no object metadata in footer + put_timestamp = utils.Timestamp(time()).internal + headers = { + 'Content-Type': 'text/plain', + 'X-Timestamp': put_timestamp, + 'Transfer-Encoding': 'chunked', + 'Expect': '100-continue', + # normally the frag index gets sent in the MIME footer (which this + # test doesn't have, see `test_multiphase_put_metadata_footer`), + # but the proxy *could* send the frag index in the headers and + # this test verifies that would work. + 'X-Object-Sysmeta-Ec-Frag-Index': '2', + 'X-Backend-Storage-Policy-Index': '1', + 'X-Backend-Obj-Content-Length': len(test_data), + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary123', + 'X-Backend-Obj-Multiphase-Commit': 'yes', + } + conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0', + 'PUT', '/a/c/o', headers=headers) + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + headers = HeaderKeyDict(resp.getheaders()) + self.assertEqual(headers['X-Obj-Multiphase-Commit'], 'yes') + + to_send = "%x\r\n%s\r\n0\r\n\r\n" % (len(test_doc), test_doc) + conn.send(to_send) + # verify 100-continue response to mark end of phase1 + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + + # send commit confirmation to start phase2 + commit_confirmation_doc = "\r\n".join(( + "X-Document: put commit", + "", + "commit_confirmation", + "--boundary123--", + )) + to_send = "%x\r\n%s\r\n0\r\n\r\n" % \ + (len(commit_confirmation_doc), commit_confirmation_doc) + conn.send(to_send) + + # verify success (2xx) to make end of phase2 + resp = conn.getresponse() + self.assertEqual(resp.status, 201) + resp.read() + resp.close() + + # verify successful object data and durable state file write + obj_basename = os.path.join( + self.devices, 'sda1', + storage_directory(diskfile.get_data_dir(POLICIES[1]), '0', + hash_path('a', 'c', 'o')), + put_timestamp) + obj_datafile = obj_basename + '#2.data' + self.assertTrue(os.path.isfile(obj_datafile)) + obj_durablefile = obj_basename + '.durable' + self.assertTrue(os.path.isfile(obj_durablefile)) + + def test_multiphase_put_draining(self): + # We want to ensure that we read the whole response body even if + # it's multipart MIME and there's document parts that we don't + # expect or understand. This'll help save our bacon if we ever jam + # more stuff in there. + in_a_timeout = [False] + + # inherit from BaseException so we get a stack trace when the test + # fails instead of just a 500 + class NotInATimeout(BaseException): + pass + + class FakeTimeout(BaseException): + def __enter__(self): + in_a_timeout[0] = True + + def __exit__(self, typ, value, tb): + in_a_timeout[0] = False + + class PickyWsgiStringIO(WsgiStringIO): + def read(self, *a, **kw): + if not in_a_timeout[0]: + raise NotInATimeout() + return WsgiStringIO.read(self, *a, **kw) + + def readline(self, *a, **kw): + if not in_a_timeout[0]: + raise NotInATimeout() + return WsgiStringIO.readline(self, *a, **kw) + + test_data = 'obj data' + footer_meta = { + "X-Object-Sysmeta-Ec-Frag-Index": "7", + "Etag": md5(test_data).hexdigest(), + } + footer_json = json.dumps(footer_meta) + footer_meta_cksum = md5(footer_json).hexdigest() + test_doc = "\r\n".join(( + "--boundary123", + "X-Document: object body", + "", + test_data, + "--boundary123", + "X-Document: object metadata", + "Content-MD5: " + footer_meta_cksum, + "", + footer_json, + "--boundary123", + "X-Document: we got cleverer", + "", + "stuff stuff meaningless stuuuuuuuuuuff", + "--boundary123", + "X-Document: we got even cleverer; can you believe it?", + "Waneshaft: ambifacient lunar", + "Casing: malleable logarithmic", + "", + "potato potato potato potato potato potato potato", + "--boundary123--" + )) + + # phase1 - PUT request with object metadata in footer and + # multiphase commit conversation + put_timestamp = utils.Timestamp(time()).internal + headers = { + 'Content-Type': 'text/plain', + 'X-Timestamp': put_timestamp, + 'Transfer-Encoding': 'chunked', + 'Expect': '100-continue', + 'X-Backend-Storage-Policy-Index': '1', + 'X-Backend-Obj-Content-Length': len(test_data), + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary123', + } + wsgi_input = PickyWsgiStringIO(test_doc) + req = Request.blank( + "/sda1/0/a/c/o", + environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': wsgi_input}, + headers=headers) + + app = object_server.ObjectController(self.conf, logger=self.logger) + with mock.patch('swift.obj.server.ChunkReadTimeout', FakeTimeout): + resp = req.get_response(app) + self.assertEqual(resp.status_int, 201) # sanity check + + in_a_timeout[0] = True # so we can check without an exception + self.assertEqual(wsgi_input.read(), '') # we read all the bytes + + def test_multiphase_put_bad_commit_message(self): + # Test 2-phase commit conversation - end of 1st phase marked + # by 100-continue response from the object server, with 2nd + # phase commit confirmation being received corrupt + test_data = 'obj data' + footer_meta = { + "X-Object-Sysmeta-Ec-Frag-Index": "7", + "Etag": md5(test_data).hexdigest(), + } + footer_json = json.dumps(footer_meta) + footer_meta_cksum = md5(footer_json).hexdigest() + test_doc = "\r\n".join(( + "--boundary123", + "X-Document: object body", + "", + test_data, + "--boundary123", + "X-Document: object metadata", + "Content-MD5: " + footer_meta_cksum, + "", + footer_json, + "--boundary123", + )) + + # phase1 - PUT request with object metadata in footer and + # multiphase commit conversation + put_timestamp = utils.Timestamp(time()).internal + headers = { + 'Content-Type': 'text/plain', + 'X-Timestamp': put_timestamp, + 'Transfer-Encoding': 'chunked', + 'Expect': '100-continue', + 'X-Backend-Storage-Policy-Index': '1', + 'X-Backend-Obj-Content-Length': len(test_data), + 'X-Backend-Obj-Metadata-Footer': 'yes', + 'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary123', + 'X-Backend-Obj-Multiphase-Commit': 'yes', + } + conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0', + 'PUT', '/a/c/o', headers=headers) + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + headers = HeaderKeyDict(resp.getheaders()) + self.assertEqual(headers['X-Obj-Multiphase-Commit'], 'yes') + self.assertEqual(headers['X-Obj-Metadata-Footer'], 'yes') + + to_send = "%x\r\n%s\r\n0\r\n\r\n" % (len(test_doc), test_doc) + conn.send(to_send) + # verify 100-continue response to mark end of phase1 + resp = conn.getexpect() + self.assertEqual(resp.status, 100) + + # send commit confirmation to start phase2 + commit_confirmation_doc = "\r\n".join(( + "junkjunk", + "--boundary123--", + )) + to_send = "%x\r\n%s\r\n0\r\n\r\n" % \ + (len(commit_confirmation_doc), commit_confirmation_doc) + conn.send(to_send) + resp = conn.getresponse() + self.assertEqual(resp.status, 500) + resp.read() + resp.close() + # verify that durable file was NOT created + obj_basename = os.path.join( + self.devices, 'sda1', + storage_directory(diskfile.get_data_dir(1), '0', + hash_path('a', 'c', 'o')), + put_timestamp) + obj_datafile = obj_basename + '#7.data' + self.assertTrue(os.path.isfile(obj_datafile)) + obj_durablefile = obj_basename + '.durable' + self.assertFalse(os.path.isfile(obj_durablefile)) + +@patch_policies class TestZeroCopy(unittest.TestCase): """Test the object server's zero-copy functionality""" diff --git a/test/unit/obj/test_ssync_receiver.py b/test/unit/obj/test_ssync_receiver.py index 9af76185b..4a030c821 100644 --- a/test/unit/obj/test_ssync_receiver.py +++ b/test/unit/obj/test_ssync_receiver.py @@ -27,6 +27,7 @@ from swift.common import constraints from swift.common import exceptions from swift.common import swob from swift.common import utils +from swift.common.storage_policy import POLICIES from swift.obj import diskfile from swift.obj import server from swift.obj import ssync_receiver @@ -34,6 +35,7 @@ from swift.obj import ssync_receiver from test import unit +@unit.patch_policies() class TestReceiver(unittest.TestCase): def setUp(self): @@ -46,12 +48,12 @@ class TestReceiver(unittest.TestCase): self.testdir = os.path.join( tempfile.mkdtemp(), 'tmp_test_ssync_receiver') utils.mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) - conf = { + self.conf = { 'devices': self.testdir, 'mount_check': 'false', 'replication_one_per_device': 'false', 'log_requests': 'false'} - self.controller = server.ObjectController(conf) + self.controller = server.ObjectController(self.conf) self.controller.bytes_per_sync = 1 self.account1 = 'a' @@ -91,14 +93,14 @@ class TestReceiver(unittest.TestCase): lines.append(line) return lines - def test_REPLICATION_semaphore_locked(self): + def test_SSYNC_semaphore_locked(self): with mock.patch.object( self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: self.controller.logger = mock.MagicMock() mocked_replication_semaphore.acquire.return_value = False req = swob.Request.blank( - '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) + '/device/partition', environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), @@ -109,13 +111,13 @@ class TestReceiver(unittest.TestCase): self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) - def test_REPLICATION_calls_replication_lock(self): + def test_SSYNC_calls_replication_lock(self): with mock.patch.object( - self.controller._diskfile_mgr, 'replication_lock') as \ - mocked_replication_lock: + self.controller._diskfile_router[POLICIES.legacy], + 'replication_lock') as mocked_replication_lock: req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') @@ -130,7 +132,7 @@ class TestReceiver(unittest.TestCase): def test_Receiver_with_default_storage_policy(self): req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') @@ -140,13 +142,15 @@ class TestReceiver(unittest.TestCase): body_lines, [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) - self.assertEqual(rcvr.policy_idx, 0) + self.assertEqual(rcvr.policy, POLICIES[0]) - @unit.patch_policies() def test_Receiver_with_storage_policy_index_header(self): + # update router post policy patch + self.controller._diskfile_router = diskfile.DiskFileRouter( + self.conf, self.controller.logger) req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION', + environ={'REQUEST_METHOD': 'SSYNC', 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': '1'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' @@ -157,19 +161,58 @@ class TestReceiver(unittest.TestCase): body_lines, [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) - self.assertEqual(rcvr.policy_idx, 1) + self.assertEqual(rcvr.policy, POLICIES[1]) + self.assertEqual(rcvr.frag_index, None) - def test_REPLICATION_replication_lock_fail(self): + def test_Receiver_with_bad_storage_policy_index_header(self): + valid_indices = sorted([int(policy) for policy in POLICIES]) + bad_index = valid_indices[-1] + 1 + req = swob.Request.blank( + '/sda1/1', + environ={'REQUEST_METHOD': 'SSYNC', + 'HTTP_X_BACKEND_SSYNC_FRAG_INDEX': '0', + 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': bad_index}, + body=':MISSING_CHECK: START\r\n' + ':MISSING_CHECK: END\r\n' + ':UPDATES: START\r\n:UPDATES: END\r\n') + self.controller.logger = mock.MagicMock() + receiver = ssync_receiver.Receiver(self.controller, req) + body_lines = [chunk.strip() for chunk in receiver() if chunk.strip()] + self.assertEqual(body_lines, [":ERROR: 503 'No policy with index 2'"]) + + @unit.patch_policies() + def test_Receiver_with_frag_index_header(self): + # update router post policy patch + self.controller._diskfile_router = diskfile.DiskFileRouter( + self.conf, self.controller.logger) + req = swob.Request.blank( + '/sda1/1', + environ={'REQUEST_METHOD': 'SSYNC', + 'HTTP_X_BACKEND_SSYNC_FRAG_INDEX': '7', + 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': '1'}, + body=':MISSING_CHECK: START\r\n' + ':MISSING_CHECK: END\r\n' + ':UPDATES: START\r\n:UPDATES: END\r\n') + rcvr = ssync_receiver.Receiver(self.controller, req) + body_lines = [chunk.strip() for chunk in rcvr() if chunk.strip()] + self.assertEqual( + body_lines, + [':MISSING_CHECK: START', ':MISSING_CHECK: END', + ':UPDATES: START', ':UPDATES: END']) + self.assertEqual(rcvr.policy, POLICIES[1]) + self.assertEqual(rcvr.frag_index, 7) + + def test_SSYNC_replication_lock_fail(self): def _mock(path): with exceptions.ReplicationLockTimeout(0.01, '/somewhere/' + path): eventlet.sleep(0.05) with mock.patch.object( - self.controller._diskfile_mgr, 'replication_lock', _mock): - self.controller._diskfile_mgr + self.controller._diskfile_router[POLICIES.legacy], + 'replication_lock', _mock): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') @@ -178,19 +221,19 @@ class TestReceiver(unittest.TestCase): self.body_lines(resp.body), [":ERROR: 0 '0.01 seconds: /somewhere/sda1'"]) self.controller.logger.debug.assert_called_once_with( - 'None/sda1/1 REPLICATION LOCK TIMEOUT: 0.01 seconds: ' + 'None/sda1/1 SSYNC LOCK TIMEOUT: 0.01 seconds: ' '/somewhere/sda1') - def test_REPLICATION_initial_path(self): + def test_SSYNC_initial_path(self): with mock.patch.object( self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: req = swob.Request.blank( - '/device', environ={'REQUEST_METHOD': 'REPLICATION'}) + '/device', environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), - [":ERROR: 0 'Invalid path: /device'"]) + [":ERROR: 400 'Invalid path: /device'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(mocked_replication_semaphore.acquire.called) self.assertFalse(mocked_replication_semaphore.release.called) @@ -199,11 +242,11 @@ class TestReceiver(unittest.TestCase): self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: req = swob.Request.blank( - '/device/', environ={'REQUEST_METHOD': 'REPLICATION'}) + '/device/', environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), - [":ERROR: 0 'Invalid path: /device/'"]) + [":ERROR: 400 'Invalid path: /device/'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(mocked_replication_semaphore.acquire.called) self.assertFalse(mocked_replication_semaphore.release.called) @@ -212,7 +255,7 @@ class TestReceiver(unittest.TestCase): self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: req = swob.Request.blank( - '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) + '/device/partition', environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), @@ -226,28 +269,29 @@ class TestReceiver(unittest.TestCase): mocked_replication_semaphore: req = swob.Request.blank( '/device/partition/junk', - environ={'REQUEST_METHOD': 'REPLICATION'}) + environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), - [":ERROR: 0 'Invalid path: /device/partition/junk'"]) + [":ERROR: 400 'Invalid path: /device/partition/junk'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(mocked_replication_semaphore.acquire.called) self.assertFalse(mocked_replication_semaphore.release.called) - def test_REPLICATION_mount_check(self): + def test_SSYNC_mount_check(self): with contextlib.nested( mock.patch.object( self.controller, 'replication_semaphore'), mock.patch.object( - self.controller._diskfile_mgr, 'mount_check', False), + self.controller._diskfile_router[POLICIES.legacy], + 'mount_check', False), mock.patch.object( constraints, 'check_mount', return_value=False)) as ( mocked_replication_semaphore, mocked_mount_check, mocked_check_mount): req = swob.Request.blank( - '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) + '/device/partition', environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), @@ -259,14 +303,15 @@ class TestReceiver(unittest.TestCase): mock.patch.object( self.controller, 'replication_semaphore'), mock.patch.object( - self.controller._diskfile_mgr, 'mount_check', True), + self.controller._diskfile_router[POLICIES.legacy], + 'mount_check', True), mock.patch.object( constraints, 'check_mount', return_value=False)) as ( mocked_replication_semaphore, mocked_mount_check, mocked_check_mount): req = swob.Request.blank( - '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) + '/device/partition', environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), @@ -275,21 +320,23 @@ class TestReceiver(unittest.TestCase): "device</p></html>'"]) self.assertEqual(resp.status_int, 200) mocked_check_mount.assert_called_once_with( - self.controller._diskfile_mgr.devices, 'device') + self.controller._diskfile_router[POLICIES.legacy].devices, + 'device') mocked_check_mount.reset_mock() mocked_check_mount.return_value = True req = swob.Request.blank( - '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) + '/device/partition', environ={'REQUEST_METHOD': 'SSYNC'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':ERROR: 0 "Looking for :MISSING_CHECK: START got \'\'"']) self.assertEqual(resp.status_int, 200) mocked_check_mount.assert_called_once_with( - self.controller._diskfile_mgr.devices, 'device') + self.controller._diskfile_router[POLICIES.legacy].devices, + 'device') - def test_REPLICATION_Exception(self): + def test_SSYNC_Exception(self): class _Wrapper(StringIO.StringIO): @@ -306,7 +353,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\nBad content is here') req.remote_addr = '1.2.3.4' @@ -324,7 +371,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger.exception.assert_called_once_with( '1.2.3.4/device/partition EXCEPTION in replication.Receiver') - def test_REPLICATION_Exception_Exception(self): + def test_SSYNC_Exception_Exception(self): class _Wrapper(StringIO.StringIO): @@ -341,7 +388,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\nBad content is here') req.remote_addr = mock.MagicMock() @@ -384,7 +431,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' 'hash ts\r\n' ':MISSING_CHECK: END\r\n' @@ -426,7 +473,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' 'hash ts\r\n' ':MISSING_CHECK: END\r\n' @@ -448,7 +495,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') @@ -466,7 +513,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' @@ -484,9 +531,36 @@ class TestReceiver(unittest.TestCase): self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) + def test_MISSING_CHECK_extra_line_parts(self): + # check that rx tolerates extra parts in missing check lines to + # allow for protocol upgrades + extra_1 = 'extra' + extra_2 = 'multiple extra parts' + self.controller.logger = mock.MagicMock() + req = swob.Request.blank( + '/sda1/1', + environ={'REQUEST_METHOD': 'SSYNC'}, + body=':MISSING_CHECK: START\r\n' + + self.hash1 + ' ' + self.ts1 + ' ' + extra_1 + '\r\n' + + self.hash2 + ' ' + self.ts2 + ' ' + extra_2 + '\r\n' + ':MISSING_CHECK: END\r\n' + ':UPDATES: START\r\n:UPDATES: END\r\n') + resp = req.get_response(self.controller) + self.assertEqual( + self.body_lines(resp.body), + [':MISSING_CHECK: START', + self.hash1, + self.hash2, + ':MISSING_CHECK: END', + ':UPDATES: START', ':UPDATES: END']) + self.assertEqual(resp.status_int, 200) + self.assertFalse(self.controller.logger.error.called) + self.assertFalse(self.controller.logger.exception.called) + def test_MISSING_CHECK_have_one_exact(self): object_dir = utils.storage_directory( - os.path.join(self.testdir, 'sda1', diskfile.get_data_dir(0)), + os.path.join(self.testdir, 'sda1', + diskfile.get_data_dir(POLICIES[0])), '1', self.hash1) utils.mkdirs(object_dir) fp = open(os.path.join(object_dir, self.ts1 + '.data'), 'w+') @@ -498,7 +572,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' @@ -515,10 +589,13 @@ class TestReceiver(unittest.TestCase): self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) - @unit.patch_policies def test_MISSING_CHECK_storage_policy(self): + # update router post policy patch + self.controller._diskfile_router = diskfile.DiskFileRouter( + self.conf, self.controller.logger) object_dir = utils.storage_directory( - os.path.join(self.testdir, 'sda1', diskfile.get_data_dir(1)), + os.path.join(self.testdir, 'sda1', + diskfile.get_data_dir(POLICIES[1])), '1', self.hash1) utils.mkdirs(object_dir) fp = open(os.path.join(object_dir, self.ts1 + '.data'), 'w+') @@ -530,7 +607,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION', + environ={'REQUEST_METHOD': 'SSYNC', 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': '1'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + @@ -550,7 +627,8 @@ class TestReceiver(unittest.TestCase): def test_MISSING_CHECK_have_one_newer(self): object_dir = utils.storage_directory( - os.path.join(self.testdir, 'sda1', diskfile.get_data_dir(0)), + os.path.join(self.testdir, 'sda1', + diskfile.get_data_dir(POLICIES[0])), '1', self.hash1) utils.mkdirs(object_dir) newer_ts1 = utils.normalize_timestamp(float(self.ts1) + 1) @@ -564,7 +642,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' @@ -583,7 +661,8 @@ class TestReceiver(unittest.TestCase): def test_MISSING_CHECK_have_one_older(self): object_dir = utils.storage_directory( - os.path.join(self.testdir, 'sda1', diskfile.get_data_dir(0)), + os.path.join(self.testdir, 'sda1', + diskfile.get_data_dir(POLICIES[0])), '1', self.hash1) utils.mkdirs(object_dir) older_ts1 = utils.normalize_timestamp(float(self.ts1) - 1) @@ -597,7 +676,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' @@ -639,7 +718,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -686,7 +765,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -729,7 +808,7 @@ class TestReceiver(unittest.TestCase): mock_shutdown_safe, mock_delete): req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -751,7 +830,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'bad_subrequest_line\r\n') @@ -770,7 +849,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -790,7 +869,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n') @@ -807,7 +886,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -824,7 +903,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -843,7 +922,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n' @@ -861,7 +940,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -879,7 +958,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n\r\n') @@ -896,7 +975,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n' @@ -926,7 +1005,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' @@ -949,7 +1028,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' @@ -975,7 +1054,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' @@ -1003,7 +1082,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' @@ -1036,7 +1115,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n' @@ -1072,8 +1151,10 @@ class TestReceiver(unittest.TestCase): 'content-encoding specialty-header')}) self.assertEqual(req.read_body, '1') - @unit.patch_policies() def test_UPDATES_with_storage_policy(self): + # update router post policy patch + self.controller._diskfile_router = diskfile.DiskFileRouter( + self.conf, self.controller.logger) _PUT_request = [None] @server.public @@ -1086,7 +1167,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION', + environ={'REQUEST_METHOD': 'SSYNC', 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': '1'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' @@ -1135,7 +1216,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' @@ -1170,7 +1251,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'BONK /a/c/o\r\n' @@ -1206,7 +1287,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o1\r\n' @@ -1317,7 +1398,7 @@ class TestReceiver(unittest.TestCase): self.assertEqual(_requests, []) def test_UPDATES_subreq_does_not_read_all(self): - # This tests that if a REPLICATION subrequest fails and doesn't read + # This tests that if a SSYNC subrequest fails and doesn't read # all the subrequest body that it will read and throw away the rest of # the body before moving on to the next subrequest. # If you comment out the part in ssync_receiver where it does: @@ -1346,7 +1427,7 @@ class TestReceiver(unittest.TestCase): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', - environ={'REQUEST_METHOD': 'REPLICATION'}, + environ={'REQUEST_METHOD': 'SSYNC'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o1\r\n' diff --git a/test/unit/obj/test_ssync_sender.py b/test/unit/obj/test_ssync_sender.py index 87efd64cc..42bd610eb 100644 --- a/test/unit/obj/test_ssync_sender.py +++ b/test/unit/obj/test_ssync_sender.py @@ -22,18 +22,24 @@ import time import unittest import eventlet +import itertools import mock from swift.common import exceptions, utils -from swift.obj import ssync_sender, diskfile +from swift.common.storage_policy import POLICIES +from swift.common.exceptions import DiskFileNotExist, DiskFileError, \ + DiskFileDeleted +from swift.common.swob import Request +from swift.common.utils import Timestamp, FileLikeIter +from swift.obj import ssync_sender, diskfile, server, ssync_receiver +from swift.obj.reconstructor import RebuildingECDiskFileStream -from test.unit import DebugLogger, patch_policies +from test.unit import debug_logger, patch_policies class FakeReplicator(object): - - def __init__(self, testdir): - self.logger = mock.MagicMock() + def __init__(self, testdir, policy=None): + self.logger = debug_logger('test-ssync-sender') self.conn_timeout = 1 self.node_timeout = 2 self.http_timeout = 3 @@ -43,7 +49,9 @@ class FakeReplicator(object): 'devices': testdir, 'mount_check': 'false', } - self._diskfile_mgr = diskfile.DiskFileManager(conf, DebugLogger()) + policy = POLICIES.default if policy is None else policy + self._diskfile_router = diskfile.DiskFileRouter(conf, self.logger) + self._diskfile_mgr = self._diskfile_router[policy] class NullBufferedHTTPConnection(object): @@ -90,39 +98,49 @@ class FakeConnection(object): self.closed = True -class TestSender(unittest.TestCase): - +class BaseTestSender(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() self.testdir = os.path.join(self.tmpdir, 'tmp_test_ssync_sender') - self.replicator = FakeReplicator(self.testdir) - self.sender = ssync_sender.Sender(self.replicator, None, None, None) + utils.mkdirs(os.path.join(self.testdir, 'dev')) + self.daemon = FakeReplicator(self.testdir) + self.sender = ssync_sender.Sender(self.daemon, None, None, None) def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=1) + shutil.rmtree(self.tmpdir, ignore_errors=True) def _make_open_diskfile(self, device='dev', partition='9', account='a', container='c', obj='o', body='test', - extra_metadata=None, policy_idx=0): + extra_metadata=None, policy=None, + frag_index=None, timestamp=None, df_mgr=None): + policy = policy or POLICIES.legacy object_parts = account, container, obj - req_timestamp = utils.normalize_timestamp(time.time()) - df = self.sender.daemon._diskfile_mgr.get_diskfile( - device, partition, *object_parts, policy_idx=policy_idx) + timestamp = Timestamp(time.time()) if timestamp is None else timestamp + if df_mgr is None: + df_mgr = self.daemon._diskfile_router[policy] + df = df_mgr.get_diskfile( + device, partition, *object_parts, policy=policy, + frag_index=frag_index) content_length = len(body) etag = hashlib.md5(body).hexdigest() with df.create() as writer: writer.write(body) metadata = { - 'X-Timestamp': req_timestamp, - 'Content-Length': content_length, + 'X-Timestamp': timestamp.internal, + 'Content-Length': str(content_length), 'ETag': etag, } if extra_metadata: metadata.update(extra_metadata) writer.put(metadata) + writer.commit(timestamp) df.open() return df + +@patch_policies() +class TestSender(BaseTestSender): + def test_call_catches_MessageTimeout(self): def connect(self): @@ -134,16 +152,16 @@ class TestSender(unittest.TestCase): with mock.patch.object(ssync_sender.Sender, 'connect', connect): node = dict(replication_ip='1.2.3.4', replication_port=5678, device='sda1') - job = dict(partition='9') - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + job = dict(partition='9', policy=POLICIES.legacy) + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) - call = self.replicator.logger.error.mock_calls[0] - self.assertEqual( - call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) - self.assertEqual(str(call[1][-1]), '1 second: test connect') + self.assertEquals(candidates, {}) + error_lines = self.daemon.logger.get_lines_for_level('error') + self.assertEqual(1, len(error_lines)) + self.assertEqual('1.2.3.4:5678/sda1/9 1 second: test connect', + error_lines[0]) def test_call_catches_ReplicationException(self): @@ -153,45 +171,44 @@ class TestSender(unittest.TestCase): with mock.patch.object(ssync_sender.Sender, 'connect', connect): node = dict(replication_ip='1.2.3.4', replication_port=5678, device='sda1') - job = dict(partition='9') - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + job = dict(partition='9', policy=POLICIES.legacy) + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) - call = self.replicator.logger.error.mock_calls[0] - self.assertEqual( - call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) - self.assertEqual(str(call[1][-1]), 'test connect') + self.assertEquals(candidates, {}) + error_lines = self.daemon.logger.get_lines_for_level('error') + self.assertEqual(1, len(error_lines)) + self.assertEqual('1.2.3.4:5678/sda1/9 test connect', + error_lines[0]) def test_call_catches_other_exceptions(self): node = dict(replication_ip='1.2.3.4', replication_port=5678, device='sda1') - job = dict(partition='9') - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + job = dict(partition='9', policy=POLICIES.legacy) + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] self.sender.connect = 'cause exception' success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) - call = self.replicator.logger.exception.mock_calls[0] - self.assertEqual( - call[1], - ('%s:%s/%s/%s EXCEPTION in replication.Sender', '1.2.3.4', 5678, - 'sda1', '9')) + self.assertEquals(candidates, {}) + error_lines = self.daemon.logger.get_lines_for_level('error') + for line in error_lines: + self.assertTrue(line.startswith( + '1.2.3.4:5678/sda1/9 EXCEPTION in replication.Sender:')) def test_call_catches_exception_handling_exception(self): - node = dict(replication_ip='1.2.3.4', replication_port=5678, - device='sda1') - job = None # Will cause inside exception handler to fail - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + job = node = None # Will cause inside exception handler to fail + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] self.sender.connect = 'cause exception' success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) - self.replicator.logger.exception.assert_called_once_with( - 'EXCEPTION in replication.Sender') + self.assertEquals(candidates, {}) + error_lines = self.daemon.logger.get_lines_for_level('error') + for line in error_lines: + self.assertTrue(line.startswith( + 'EXCEPTION in replication.Sender')) def test_call_calls_others(self): self.sender.suffixes = ['abc'] @@ -201,7 +218,7 @@ class TestSender(unittest.TestCase): self.sender.disconnect = mock.MagicMock() success, candidates = self.sender() self.assertTrue(success) - self.assertEquals(candidates, set()) + self.assertEquals(candidates, {}) self.sender.connect.assert_called_once_with() self.sender.missing_check.assert_called_once_with() self.sender.updates.assert_called_once_with() @@ -216,18 +233,17 @@ class TestSender(unittest.TestCase): self.sender.failures = 1 success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) + self.assertEquals(candidates, {}) self.sender.connect.assert_called_once_with() self.sender.missing_check.assert_called_once_with() self.sender.updates.assert_called_once_with() self.sender.disconnect.assert_called_once_with() - @patch_policies def test_connect(self): node = dict(replication_ip='1.2.3.4', replication_port=5678, - device='sda1') - job = dict(partition='9', policy_idx=1) - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + device='sda1', index=0) + job = dict(partition='9', policy=POLICIES[1]) + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] with mock.patch( 'swift.obj.ssync_sender.bufferedhttp.BufferedHTTPConnection' @@ -240,11 +256,12 @@ class TestSender(unittest.TestCase): mock_conn_class.assert_called_once_with('1.2.3.4:5678') expectations = { 'putrequest': [ - mock.call('REPLICATION', '/sda1/9'), + mock.call('SSYNC', '/sda1/9'), ], 'putheader': [ mock.call('Transfer-Encoding', 'chunked'), mock.call('X-Backend-Storage-Policy-Index', 1), + mock.call('X-Backend-Ssync-Frag-Index', 0), ], 'endheaders': [mock.call()], } @@ -255,10 +272,80 @@ class TestSender(unittest.TestCase): method_name, mock_method.mock_calls, expected_calls)) + def test_call(self): + def patch_sender(sender): + sender.connect = mock.MagicMock() + sender.missing_check = mock.MagicMock() + sender.updates = mock.MagicMock() + sender.disconnect = mock.MagicMock() + + node = dict(replication_ip='1.2.3.4', replication_port=5678, + device='sda1') + job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + 'frag_index': 0, + } + available_map = dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000'), + ('9d41d8cd98f00b204e9800998ecf0def', + '1380144472.22222'), + ('9d41d8cd98f00b204e9800998ecf1def', + '1380144474.44444')]) + + # no suffixes -> no work done + sender = ssync_sender.Sender( + self.daemon, node, job, [], remote_check_objs=None) + patch_sender(sender) + sender.available_map = available_map + success, candidates = sender() + self.assertTrue(success) + self.assertEqual({}, candidates) + + # all objs in sync + sender = ssync_sender.Sender( + self.daemon, node, job, ['ignored'], remote_check_objs=None) + patch_sender(sender) + sender.available_map = available_map + success, candidates = sender() + self.assertTrue(success) + self.assertEqual(available_map, candidates) + + # one obj not in sync, sync'ing faked, all objs should be in return set + wanted = '9d41d8cd98f00b204e9800998ecf0def' + sender = ssync_sender.Sender( + self.daemon, node, job, ['ignored'], + remote_check_objs=None) + patch_sender(sender) + sender.send_list = [wanted] + sender.available_map = available_map + success, candidates = sender() + self.assertTrue(success) + self.assertEqual(available_map, candidates) + + # one obj not in sync, remote check only so that obj is not sync'd + # and should not be in the return set + wanted = '9d41d8cd98f00b204e9800998ecf0def' + remote_check_objs = set(available_map.keys()) + sender = ssync_sender.Sender( + self.daemon, node, job, ['ignored'], + remote_check_objs=remote_check_objs) + patch_sender(sender) + sender.send_list = [wanted] + sender.available_map = available_map + success, candidates = sender() + self.assertTrue(success) + expected_map = dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000'), + ('9d41d8cd98f00b204e9800998ecf1def', + '1380144474.44444')]) + self.assertEqual(expected_map, candidates) + def test_call_and_missing_check(self): - def yield_hashes(device, partition, policy_index, suffixes=None): + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): if device == 'dev' and partition == '9' and suffixes == ['abc'] \ - and policy_index == 0: + and policy == POLICIES.legacy: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', @@ -269,7 +356,12 @@ class TestSender(unittest.TestCase): 'No match for %r %r %r' % (device, partition, suffixes)) self.sender.connection = FakeConnection() - self.sender.job = {'device': 'dev', 'partition': '9'} + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + 'frag_index': 0, + } self.sender.suffixes = ['abc'] self.sender.response = FakeResponse( chunk_body=( @@ -282,13 +374,14 @@ class TestSender(unittest.TestCase): self.sender.disconnect = mock.MagicMock() success, candidates = self.sender() self.assertTrue(success) - self.assertEqual(candidates, set(['9d41d8cd98f00b204e9800998ecf0abc'])) + self.assertEqual(candidates, dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000')])) self.assertEqual(self.sender.failures, 0) def test_call_and_missing_check_with_obj_list(self): - def yield_hashes(device, partition, policy_index, suffixes=None): + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): if device == 'dev' and partition == '9' and suffixes == ['abc'] \ - and policy_index == 0: + and policy == POLICIES.legacy: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', @@ -297,8 +390,13 @@ class TestSender(unittest.TestCase): else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) - job = {'device': 'dev', 'partition': '9'} - self.sender = ssync_sender.Sender(self.replicator, None, job, ['abc'], + job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + 'frag_index': 0, + } + self.sender = ssync_sender.Sender(self.daemon, None, job, ['abc'], ['9d41d8cd98f00b204e9800998ecf0abc']) self.sender.connection = FakeConnection() self.sender.response = FakeResponse( @@ -311,13 +409,14 @@ class TestSender(unittest.TestCase): self.sender.disconnect = mock.MagicMock() success, candidates = self.sender() self.assertTrue(success) - self.assertEqual(candidates, set(['9d41d8cd98f00b204e9800998ecf0abc'])) + self.assertEqual(candidates, dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000')])) self.assertEqual(self.sender.failures, 0) def test_call_and_missing_check_with_obj_list_but_required(self): - def yield_hashes(device, partition, policy_index, suffixes=None): + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): if device == 'dev' and partition == '9' and suffixes == ['abc'] \ - and policy_index == 0: + and policy == POLICIES.legacy: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', @@ -326,8 +425,13 @@ class TestSender(unittest.TestCase): else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) - job = {'device': 'dev', 'partition': '9'} - self.sender = ssync_sender.Sender(self.replicator, None, job, ['abc'], + job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + 'frag_index': 0, + } + self.sender = ssync_sender.Sender(self.daemon, None, job, ['abc'], ['9d41d8cd98f00b204e9800998ecf0abc']) self.sender.connection = FakeConnection() self.sender.response = FakeResponse( @@ -341,14 +445,14 @@ class TestSender(unittest.TestCase): self.sender.disconnect = mock.MagicMock() success, candidates = self.sender() self.assertTrue(success) - self.assertEqual(candidates, set()) + self.assertEqual(candidates, {}) def test_connect_send_timeout(self): - self.replicator.conn_timeout = 0.01 + self.daemon.conn_timeout = 0.01 node = dict(replication_ip='1.2.3.4', replication_port=5678, device='sda1') - job = dict(partition='9') - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + job = dict(partition='9', policy=POLICIES.legacy) + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] def putrequest(*args, **kwargs): @@ -359,18 +463,18 @@ class TestSender(unittest.TestCase): 'putrequest', putrequest): success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) - call = self.replicator.logger.error.mock_calls[0] - self.assertEqual( - call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) - self.assertEqual(str(call[1][-1]), '0.01 seconds: connect send') + self.assertEquals(candidates, {}) + error_lines = self.daemon.logger.get_lines_for_level('error') + for line in error_lines: + self.assertTrue(line.startswith( + '1.2.3.4:5678/sda1/9 0.01 seconds: connect send')) def test_connect_receive_timeout(self): - self.replicator.node_timeout = 0.02 + self.daemon.node_timeout = 0.02 node = dict(replication_ip='1.2.3.4', replication_port=5678, - device='sda1') - job = dict(partition='9') - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + device='sda1', index=0) + job = dict(partition='9', policy=POLICIES.legacy) + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] class FakeBufferedHTTPConnection(NullBufferedHTTPConnection): @@ -383,18 +487,18 @@ class TestSender(unittest.TestCase): FakeBufferedHTTPConnection): success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) - call = self.replicator.logger.error.mock_calls[0] - self.assertEqual( - call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) - self.assertEqual(str(call[1][-1]), '0.02 seconds: connect receive') + self.assertEquals(candidates, {}) + error_lines = self.daemon.logger.get_lines_for_level('error') + for line in error_lines: + self.assertTrue(line.startswith( + '1.2.3.4:5678/sda1/9 0.02 seconds: connect receive')) def test_connect_bad_status(self): - self.replicator.node_timeout = 0.02 + self.daemon.node_timeout = 0.02 node = dict(replication_ip='1.2.3.4', replication_port=5678, - device='sda1') - job = dict(partition='9') - self.sender = ssync_sender.Sender(self.replicator, node, job, None) + device='sda1', index=0) + job = dict(partition='9', policy=POLICIES.legacy) + self.sender = ssync_sender.Sender(self.daemon, node, job, None) self.sender.suffixes = ['abc'] class FakeBufferedHTTPConnection(NullBufferedHTTPConnection): @@ -408,11 +512,11 @@ class TestSender(unittest.TestCase): FakeBufferedHTTPConnection): success, candidates = self.sender() self.assertFalse(success) - self.assertEquals(candidates, set()) - call = self.replicator.logger.error.mock_calls[0] - self.assertEqual( - call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) - self.assertEqual(str(call[1][-1]), 'Expected status 200; got 503') + self.assertEquals(candidates, {}) + error_lines = self.daemon.logger.get_lines_for_level('error') + for line in error_lines: + self.assertTrue(line.startswith( + '1.2.3.4:5678/sda1/9 Expected status 200; got 503')) def test_readline_newline_in_buffer(self): self.sender.response_buffer = 'Has a newline already.\r\nOkay.' @@ -420,7 +524,7 @@ class TestSender(unittest.TestCase): self.assertEqual(self.sender.response_buffer, 'Okay.') def test_readline_buffer_exceeds_network_chunk_size_somehow(self): - self.replicator.network_chunk_size = 2 + self.daemon.network_chunk_size = 2 self.sender.response_buffer = '1234567890' self.assertEqual(self.sender.readline(), '1234567890') self.assertEqual(self.sender.response_buffer, '') @@ -473,16 +577,21 @@ class TestSender(unittest.TestCase): self.assertRaises(exceptions.MessageTimeout, self.sender.missing_check) def test_missing_check_has_empty_suffixes(self): - def yield_hashes(device, partition, policy_idx, suffixes=None): - if (device != 'dev' or partition != '9' or policy_idx != 0 or + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): + if (device != 'dev' or partition != '9' or + policy != POLICIES.legacy or suffixes != ['abc', 'def']): yield # Just here to make this a generator raise Exception( 'No match for %r %r %r %r' % (device, partition, - policy_idx, suffixes)) + policy, suffixes)) self.sender.connection = FakeConnection() - self.sender.job = {'device': 'dev', 'partition': '9'} + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + } self.sender.suffixes = ['abc', 'def'] self.sender.response = FakeResponse( chunk_body=( @@ -495,11 +604,12 @@ class TestSender(unittest.TestCase): '17\r\n:MISSING_CHECK: START\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') self.assertEqual(self.sender.send_list, []) - self.assertEqual(self.sender.available_set, set()) + self.assertEqual(self.sender.available_map, {}) def test_missing_check_has_suffixes(self): - def yield_hashes(device, partition, policy_idx, suffixes=None): - if (device == 'dev' and partition == '9' and policy_idx == 0 and + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): + if (device == 'dev' and partition == '9' and + policy == POLICIES.legacy and suffixes == ['abc', 'def']): yield ( '/srv/node/dev/objects/9/abc/' @@ -519,10 +629,14 @@ class TestSender(unittest.TestCase): else: raise Exception( 'No match for %r %r %r %r' % (device, partition, - policy_idx, suffixes)) + policy, suffixes)) self.sender.connection = FakeConnection() - self.sender.job = {'device': 'dev', 'partition': '9'} + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + } self.sender.suffixes = ['abc', 'def'] self.sender.response = FakeResponse( chunk_body=( @@ -538,14 +652,15 @@ class TestSender(unittest.TestCase): '33\r\n9d41d8cd98f00b204e9800998ecf1def 1380144474.44444\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') self.assertEqual(self.sender.send_list, []) - candidates = ['9d41d8cd98f00b204e9800998ecf0abc', - '9d41d8cd98f00b204e9800998ecf0def', - '9d41d8cd98f00b204e9800998ecf1def'] - self.assertEqual(self.sender.available_set, set(candidates)) + candidates = [('9d41d8cd98f00b204e9800998ecf0abc', '1380144470.00000'), + ('9d41d8cd98f00b204e9800998ecf0def', '1380144472.22222'), + ('9d41d8cd98f00b204e9800998ecf1def', '1380144474.44444')] + self.assertEqual(self.sender.available_map, dict(candidates)) def test_missing_check_far_end_disconnect(self): - def yield_hashes(device, partition, policy_idx, suffixes=None): - if (device == 'dev' and partition == '9' and policy_idx == 0 and + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): + if (device == 'dev' and partition == '9' and + policy == POLICIES.legacy and suffixes == ['abc']): yield ( '/srv/node/dev/objects/9/abc/' @@ -555,10 +670,14 @@ class TestSender(unittest.TestCase): else: raise Exception( 'No match for %r %r %r %r' % (device, partition, - policy_idx, suffixes)) + policy, suffixes)) self.sender.connection = FakeConnection() - self.sender.job = {'device': 'dev', 'partition': '9'} + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + } self.sender.suffixes = ['abc'] self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.response = FakeResponse(chunk_body='\r\n') @@ -573,12 +692,14 @@ class TestSender(unittest.TestCase): '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') - self.assertEqual(self.sender.available_set, - set(['9d41d8cd98f00b204e9800998ecf0abc'])) + self.assertEqual(self.sender.available_map, + dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000')])) def test_missing_check_far_end_disconnect2(self): - def yield_hashes(device, partition, policy_idx, suffixes=None): - if (device == 'dev' and partition == '9' and policy_idx == 0 and + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): + if (device == 'dev' and partition == '9' and + policy == POLICIES.legacy and suffixes == ['abc']): yield ( '/srv/node/dev/objects/9/abc/' @@ -588,10 +709,14 @@ class TestSender(unittest.TestCase): else: raise Exception( 'No match for %r %r %r %r' % (device, partition, - policy_idx, suffixes)) + policy, suffixes)) self.sender.connection = FakeConnection() - self.sender.job = {'device': 'dev', 'partition': '9'} + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + } self.sender.suffixes = ['abc'] self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.response = FakeResponse( @@ -607,12 +732,14 @@ class TestSender(unittest.TestCase): '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') - self.assertEqual(self.sender.available_set, - set(['9d41d8cd98f00b204e9800998ecf0abc'])) + self.assertEqual(self.sender.available_map, + dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000')])) def test_missing_check_far_end_unexpected(self): - def yield_hashes(device, partition, policy_idx, suffixes=None): - if (device == 'dev' and partition == '9' and policy_idx == 0 and + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): + if (device == 'dev' and partition == '9' and + policy == POLICIES.legacy and suffixes == ['abc']): yield ( '/srv/node/dev/objects/9/abc/' @@ -622,10 +749,14 @@ class TestSender(unittest.TestCase): else: raise Exception( 'No match for %r %r %r %r' % (device, partition, - policy_idx, suffixes)) + policy, suffixes)) self.sender.connection = FakeConnection() - self.sender.job = {'device': 'dev', 'partition': '9'} + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + } self.sender.suffixes = ['abc'] self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.response = FakeResponse(chunk_body='OH HAI\r\n') @@ -640,12 +771,14 @@ class TestSender(unittest.TestCase): '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') - self.assertEqual(self.sender.available_set, - set(['9d41d8cd98f00b204e9800998ecf0abc'])) + self.assertEqual(self.sender.available_map, + dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000')])) def test_missing_check_send_list(self): - def yield_hashes(device, partition, policy_idx, suffixes=None): - if (device == 'dev' and partition == '9' and policy_idx == 0 and + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): + if (device == 'dev' and partition == '9' and + policy == POLICIES.legacy and suffixes == ['abc']): yield ( '/srv/node/dev/objects/9/abc/' @@ -655,10 +788,14 @@ class TestSender(unittest.TestCase): else: raise Exception( 'No match for %r %r %r %r' % (device, partition, - policy_idx, suffixes)) + policy, suffixes)) self.sender.connection = FakeConnection() - self.sender.job = {'device': 'dev', 'partition': '9'} + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + } self.sender.suffixes = ['abc'] self.sender.response = FakeResponse( chunk_body=( @@ -673,8 +810,45 @@ class TestSender(unittest.TestCase): '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') self.assertEqual(self.sender.send_list, ['0123abc']) - self.assertEqual(self.sender.available_set, - set(['9d41d8cd98f00b204e9800998ecf0abc'])) + self.assertEqual(self.sender.available_map, + dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000')])) + + def test_missing_check_extra_line_parts(self): + # check that sender tolerates extra parts in missing check + # line responses to allow for protocol upgrades + def yield_hashes(device, partition, policy, suffixes=None, **kwargs): + if (device == 'dev' and partition == '9' and + policy == POLICIES.legacy and + suffixes == ['abc']): + yield ( + '/srv/node/dev/objects/9/abc/' + '9d41d8cd98f00b204e9800998ecf0abc', + '9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000') + else: + raise Exception( + 'No match for %r %r %r %r' % (device, partition, + policy, suffixes)) + + self.sender.connection = FakeConnection() + self.sender.job = { + 'device': 'dev', + 'partition': '9', + 'policy': POLICIES.legacy, + } + self.sender.suffixes = ['abc'] + self.sender.response = FakeResponse( + chunk_body=( + ':MISSING_CHECK: START\r\n' + '0123abc extra response parts\r\n' + ':MISSING_CHECK: END\r\n')) + self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes + self.sender.missing_check() + self.assertEqual(self.sender.send_list, ['0123abc']) + self.assertEqual(self.sender.available_map, + dict([('9d41d8cd98f00b204e9800998ecf0abc', + '1380144470.00000')])) def test_updates_timeout(self): self.sender.connection = FakeConnection() @@ -742,7 +916,12 @@ class TestSender(unittest.TestCase): delete_timestamp = utils.normalize_timestamp(time.time()) df.delete(delete_timestamp) self.sender.connection = FakeConnection() - self.sender.job = {'device': device, 'partition': part} + self.sender.job = { + 'device': device, + 'partition': part, + 'policy': POLICIES.legacy, + 'frag_index': 0, + } self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() @@ -771,7 +950,12 @@ class TestSender(unittest.TestCase): delete_timestamp = utils.normalize_timestamp(time.time()) df.delete(delete_timestamp) self.sender.connection = FakeConnection() - self.sender.job = {'device': device, 'partition': part} + self.sender.job = { + 'device': device, + 'partition': part, + 'policy': POLICIES.legacy, + 'frag_index': 0, + } self.sender.node = {} self.sender.send_list = [object_hash] self.sender.response = FakeResponse( @@ -797,7 +981,12 @@ class TestSender(unittest.TestCase): object_hash = utils.hash_path(*object_parts) expected = df.get_metadata() self.sender.connection = FakeConnection() - self.sender.job = {'device': device, 'partition': part} + self.sender.job = { + 'device': device, + 'partition': part, + 'policy': POLICIES.legacy, + 'frag_index': 0, + } self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() @@ -821,18 +1010,20 @@ class TestSender(unittest.TestCase): '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') - @patch_policies def test_updates_storage_policy_index(self): device = 'dev' part = '9' object_parts = ('a', 'c', 'o') df = self._make_open_diskfile(device, part, *object_parts, - policy_idx=1) + policy=POLICIES[0]) object_hash = utils.hash_path(*object_parts) expected = df.get_metadata() self.sender.connection = FakeConnection() - self.sender.job = {'device': device, 'partition': part, - 'policy_idx': 1} + self.sender.job = { + 'device': device, + 'partition': part, + 'policy': POLICIES[0], + 'frag_index': 0} self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() @@ -847,7 +1038,7 @@ class TestSender(unittest.TestCase): self.assertEqual(path, '/a/c/o') self.assert_(isinstance(df, diskfile.DiskFile)) self.assertEqual(expected, df.get_metadata()) - self.assertEqual(os.path.join(self.testdir, 'dev/objects-1/9/', + self.assertEqual(os.path.join(self.testdir, 'dev/objects/9/', object_hash[-3:], object_hash), df._datadir) @@ -1054,5 +1245,466 @@ class TestSender(unittest.TestCase): self.assertTrue(self.sender.connection.closed) +@patch_policies(with_ec_default=True) +class TestSsync(BaseTestSender): + """ + Test interactions between sender and receiver. The basis for each test is + actual diskfile state on either side - the connection between sender and + receiver is faked. Assertions are made about the final state of the sender + and receiver diskfiles. + """ + + def make_fake_ssync_connect(self, sender, rx_obj_controller, device, + partition, policy): + trace = [] + + def add_trace(type, msg): + # record a protocol event for later analysis + if msg.strip(): + trace.append((type, msg.strip())) + + def start_response(status, headers, exc_info=None): + assert(status == '200 OK') + + class FakeConnection: + def __init__(self, trace): + self.trace = trace + self.queue = [] + self.src = FileLikeIter(self.queue) + + def send(self, msg): + msg = msg.split('\r\n', 1)[1] + msg = msg.rsplit('\r\n', 1)[0] + add_trace('tx', msg) + self.queue.append(msg) + + def close(self): + pass + + def wrap_gen(gen): + # Strip response head and tail + while True: + try: + msg = gen.next() + if msg: + add_trace('rx', msg) + msg = '%x\r\n%s\r\n' % (len(msg), msg) + yield msg + except StopIteration: + break + + def fake_connect(): + sender.connection = FakeConnection(trace) + headers = {'Transfer-Encoding': 'chunked', + 'X-Backend-Storage-Policy-Index': str(int(policy))} + env = {'REQUEST_METHOD': 'SSYNC'} + path = '/%s/%s' % (device, partition) + req = Request.blank(path, environ=env, headers=headers) + req.environ['wsgi.input'] = sender.connection.src + resp = rx_obj_controller(req.environ, start_response) + wrapped_gen = wrap_gen(resp) + sender.response = FileLikeIter(wrapped_gen) + sender.response.fp = sender.response + return fake_connect + + def setUp(self): + self.device = 'dev' + self.partition = '9' + self.tmpdir = tempfile.mkdtemp() + # sender side setup + self.tx_testdir = os.path.join(self.tmpdir, 'tmp_test_ssync_sender') + utils.mkdirs(os.path.join(self.tx_testdir, self.device)) + self.daemon = FakeReplicator(self.tx_testdir) + + # rx side setup + self.rx_testdir = os.path.join(self.tmpdir, 'tmp_test_ssync_receiver') + utils.mkdirs(os.path.join(self.rx_testdir, self.device)) + conf = { + 'devices': self.rx_testdir, + 'mount_check': 'false', + 'replication_one_per_device': 'false', + 'log_requests': 'false'} + self.rx_controller = server.ObjectController(conf) + self.orig_ensure_flush = ssync_receiver.Receiver._ensure_flush + ssync_receiver.Receiver._ensure_flush = lambda *args: '' + self.ts_iter = (Timestamp(t) + for t in itertools.count(int(time.time()))) + + def tearDown(self): + if self.orig_ensure_flush: + ssync_receiver.Receiver._ensure_flush = self.orig_ensure_flush + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def _create_ondisk_files(self, df_mgr, obj_name, policy, timestamp, + frag_indexes=None): + frag_indexes = [] if frag_indexes is None else frag_indexes + metadata = {'Content-Type': 'plain/text'} + diskfiles = [] + for frag_index in frag_indexes: + object_data = '/a/c/%s___%s' % (obj_name, frag_index) + if frag_index is not None: + metadata['X-Object-Sysmeta-Ec-Frag-Index'] = str(frag_index) + df = self._make_open_diskfile( + device=self.device, partition=self.partition, account='a', + container='c', obj=obj_name, body=object_data, + extra_metadata=metadata, timestamp=timestamp, policy=policy, + frag_index=frag_index, df_mgr=df_mgr) + # sanity checks + listing = os.listdir(df._datadir) + self.assertTrue(listing) + for filename in listing: + self.assertTrue(filename.startswith(timestamp.internal)) + diskfiles.append(df) + return diskfiles + + def _open_tx_diskfile(self, obj_name, policy, frag_index=None): + df_mgr = self.daemon._diskfile_router[policy] + df = df_mgr.get_diskfile( + self.device, self.partition, account='a', container='c', + obj=obj_name, policy=policy, frag_index=frag_index) + df.open() + return df + + def _open_rx_diskfile(self, obj_name, policy, frag_index=None): + df = self.rx_controller.get_diskfile( + self.device, self.partition, 'a', 'c', obj_name, policy=policy, + frag_index=frag_index) + df.open() + return df + + def _verify_diskfile_sync(self, tx_df, rx_df, frag_index): + # verify that diskfiles' metadata match + # sanity check, they are not the same ondisk files! + self.assertNotEqual(tx_df._datadir, rx_df._datadir) + rx_metadata = dict(rx_df.get_metadata()) + for k, v in tx_df.get_metadata().iteritems(): + self.assertEqual(v, rx_metadata.pop(k)) + # ugh, ssync duplicates ETag with Etag so have to clear it out here + if 'Etag' in rx_metadata: + rx_metadata.pop('Etag') + self.assertFalse(rx_metadata) + if frag_index: + rx_metadata = rx_df.get_metadata() + fi_key = 'X-Object-Sysmeta-Ec-Frag-Index' + self.assertTrue(fi_key in rx_metadata) + self.assertEqual(frag_index, int(rx_metadata[fi_key])) + + def _analyze_trace(self, trace): + """ + Parse protocol trace captured by fake connection, making some + assertions along the way, and return results as a dict of form: + results = {'tx_missing': <list of messages>, + 'rx_missing': <list of messages>, + 'tx_updates': <list of subreqs>, + 'rx_updates': <list of messages>} + + Each subreq is a dict with keys: 'method', 'path', 'headers', 'body' + """ + def tx_missing(results, line): + self.assertEqual('tx', line[0]) + results['tx_missing'].append(line[1]) + + def rx_missing(results, line): + self.assertEqual('rx', line[0]) + parts = line[1].split('\r\n') + for part in parts: + results['rx_missing'].append(part) + + def tx_updates(results, line): + self.assertEqual('tx', line[0]) + subrequests = results['tx_updates'] + if line[1].startswith(('PUT', 'DELETE')): + parts = line[1].split('\r\n') + method, path = parts[0].split() + subreq = {'method': method, 'path': path, 'req': line[1], + 'headers': parts[1:]} + subrequests.append(subreq) + else: + self.assertTrue(subrequests) + body = (subrequests[-1]).setdefault('body', '') + body += line[1] + subrequests[-1]['body'] = body + + def rx_updates(results, line): + self.assertEqual('rx', line[0]) + results.setdefault['rx_updates'].append(line[1]) + + def unexpected(results, line): + results.setdefault('unexpected', []).append(line) + + # each trace line is a tuple of ([tx|rx], msg) + handshakes = iter([(('tx', ':MISSING_CHECK: START'), tx_missing), + (('tx', ':MISSING_CHECK: END'), unexpected), + (('rx', ':MISSING_CHECK: START'), rx_missing), + (('rx', ':MISSING_CHECK: END'), unexpected), + (('tx', ':UPDATES: START'), tx_updates), + (('tx', ':UPDATES: END'), unexpected), + (('rx', ':UPDATES: START'), rx_updates), + (('rx', ':UPDATES: END'), unexpected)]) + expect_handshake = handshakes.next() + phases = ('tx_missing', 'rx_missing', 'tx_updates', 'rx_updates') + results = dict((k, []) for k in phases) + handler = unexpected + lines = list(trace) + lines.reverse() + while lines: + line = lines.pop() + if line == expect_handshake[0]: + handler = expect_handshake[1] + try: + expect_handshake = handshakes.next() + except StopIteration: + # should be the last line + self.assertFalse( + lines, 'Unexpected trailing lines %s' % lines) + continue + handler(results, line) + + try: + # check all handshakes occurred + missed = handshakes.next() + self.fail('Handshake %s not found' % str(missed[0])) + except StopIteration: + pass + # check no message outside of a phase + self.assertFalse(results.get('unexpected'), + 'Message outside of a phase: %s' % results.get(None)) + return results + + def _verify_ondisk_files(self, tx_objs, policy, rx_node_index): + # verify tx and rx files that should be in sync + for o_name, diskfiles in tx_objs.iteritems(): + for tx_df in diskfiles: + frag_index = tx_df._frag_index + if frag_index == rx_node_index: + # this frag_index should have been sync'd, + # check rx file is ok + rx_df = self._open_rx_diskfile(o_name, policy, frag_index) + self._verify_diskfile_sync(tx_df, rx_df, frag_index) + expected_body = '/a/c/%s___%s' % (o_name, rx_node_index) + actual_body = ''.join([chunk for chunk in rx_df.reader()]) + self.assertEqual(expected_body, actual_body) + else: + # this frag_index should not have been sync'd, + # check no rx file, + self.assertRaises(DiskFileNotExist, + self._open_rx_diskfile, + o_name, policy, frag_index=frag_index) + # check tx file still intact - ssync does not do any cleanup! + self._open_tx_diskfile(o_name, policy, frag_index) + + def _verify_tombstones(self, tx_objs, policy): + # verify tx and rx tombstones that should be in sync + for o_name, diskfiles in tx_objs.iteritems(): + for tx_df_ in diskfiles: + try: + self._open_tx_diskfile(o_name, policy) + self.fail('DiskFileDeleted expected') + except DiskFileDeleted as exc: + tx_delete_time = exc.timestamp + try: + self._open_rx_diskfile(o_name, policy) + self.fail('DiskFileDeleted expected') + except DiskFileDeleted as exc: + rx_delete_time = exc.timestamp + self.assertEqual(tx_delete_time, rx_delete_time) + + def test_handoff_fragment_revert(self): + # test that a sync_revert type job does send the correct frag archives + # to the receiver, and that those frag archives are then removed from + # local node. + policy = POLICIES.default + rx_node_index = 0 + tx_node_index = 1 + frag_index = rx_node_index + + # create sender side diskfiles... + tx_objs = {} + rx_objs = {} + tx_tombstones = {} + tx_df_mgr = self.daemon._diskfile_router[policy] + rx_df_mgr = self.rx_controller._diskfile_router[policy] + # o1 has primary and handoff fragment archives + t1 = self.ts_iter.next() + tx_objs['o1'] = self._create_ondisk_files( + tx_df_mgr, 'o1', policy, t1, (rx_node_index, tx_node_index)) + # o2 only has primary + t2 = self.ts_iter.next() + tx_objs['o2'] = self._create_ondisk_files( + tx_df_mgr, 'o2', policy, t2, (tx_node_index,)) + # o3 only has handoff + t3 = self.ts_iter.next() + tx_objs['o3'] = self._create_ondisk_files( + tx_df_mgr, 'o3', policy, t3, (rx_node_index,)) + # o4 primary and handoff fragment archives on tx, handoff in sync on rx + t4 = self.ts_iter.next() + tx_objs['o4'] = self._create_ondisk_files( + tx_df_mgr, 'o4', policy, t4, (tx_node_index, rx_node_index,)) + rx_objs['o4'] = self._create_ondisk_files( + rx_df_mgr, 'o4', policy, t4, (rx_node_index,)) + # o5 is a tombstone, missing on receiver + t5 = self.ts_iter.next() + tx_tombstones['o5'] = self._create_ondisk_files( + tx_df_mgr, 'o5', policy, t5, (tx_node_index,)) + tx_tombstones['o5'][0].delete(t5) + + suffixes = set() + for diskfiles in (tx_objs.values() + tx_tombstones.values()): + for df in diskfiles: + suffixes.add(os.path.basename(os.path.dirname(df._datadir))) + + # create ssync sender instance... + job = {'device': self.device, + 'partition': self.partition, + 'policy': policy, + 'frag_index': frag_index, + 'purge': True} + node = {'index': rx_node_index} + self.sender = ssync_sender.Sender(self.daemon, node, job, suffixes) + # fake connection from tx to rx... + self.sender.connect = self.make_fake_ssync_connect( + self.sender, self.rx_controller, self.device, self.partition, + policy) + + # run the sync protocol... + self.sender() + + # verify protocol + results = self._analyze_trace(self.sender.connection.trace) + # sender has handoff frags for o1, o3 and o4 and ts for o5 + self.assertEqual(4, len(results['tx_missing'])) + # receiver is missing frags for o1, o3 and ts for o5 + self.assertEqual(3, len(results['rx_missing'])) + self.assertEqual(3, len(results['tx_updates'])) + self.assertFalse(results['rx_updates']) + sync_paths = [] + for subreq in results.get('tx_updates'): + if subreq.get('method') == 'PUT': + self.assertTrue( + 'X-Object-Sysmeta-Ec-Frag-Index: %s' % rx_node_index + in subreq.get('headers')) + expected_body = '%s___%s' % (subreq['path'], rx_node_index) + self.assertEqual(expected_body, subreq['body']) + elif subreq.get('method') == 'DELETE': + self.assertEqual('/a/c/o5', subreq['path']) + sync_paths.append(subreq.get('path')) + self.assertEqual(['/a/c/o1', '/a/c/o3', '/a/c/o5'], sorted(sync_paths)) + + # verify on disk files... + self._verify_ondisk_files(tx_objs, policy, rx_node_index) + self._verify_tombstones(tx_tombstones, policy) + + def test_fragment_sync(self): + # check that a sync_only type job does call reconstructor to build a + # diskfile to send, and continues making progress despite an error + # when building one diskfile + policy = POLICIES.default + rx_node_index = 0 + tx_node_index = 1 + # for a sync job we iterate over frag index that belongs on local node + frag_index = tx_node_index + + # create sender side diskfiles... + tx_objs = {} + tx_tombstones = {} + rx_objs = {} + tx_df_mgr = self.daemon._diskfile_router[policy] + rx_df_mgr = self.rx_controller._diskfile_router[policy] + # o1 only has primary + t1 = self.ts_iter.next() + tx_objs['o1'] = self._create_ondisk_files( + tx_df_mgr, 'o1', policy, t1, (tx_node_index,)) + # o2 only has primary + t2 = self.ts_iter.next() + tx_objs['o2'] = self._create_ondisk_files( + tx_df_mgr, 'o2', policy, t2, (tx_node_index,)) + # o3 only has primary + t3 = self.ts_iter.next() + tx_objs['o3'] = self._create_ondisk_files( + tx_df_mgr, 'o3', policy, t3, (tx_node_index,)) + # o4 primary fragment archives on tx, handoff in sync on rx + t4 = self.ts_iter.next() + tx_objs['o4'] = self._create_ondisk_files( + tx_df_mgr, 'o4', policy, t4, (tx_node_index,)) + rx_objs['o4'] = self._create_ondisk_files( + rx_df_mgr, 'o4', policy, t4, (rx_node_index,)) + # o5 is a tombstone, missing on receiver + t5 = self.ts_iter.next() + tx_tombstones['o5'] = self._create_ondisk_files( + tx_df_mgr, 'o5', policy, t5, (tx_node_index,)) + tx_tombstones['o5'][0].delete(t5) + + suffixes = set() + for diskfiles in (tx_objs.values() + tx_tombstones.values()): + for df in diskfiles: + suffixes.add(os.path.basename(os.path.dirname(df._datadir))) + + reconstruct_fa_calls = [] + + def fake_reconstruct_fa(job, node, metadata): + reconstruct_fa_calls.append((job, node, policy, metadata)) + if len(reconstruct_fa_calls) == 2: + # simulate second reconstruct failing + raise DiskFileError + content = '%s___%s' % (metadata['name'], rx_node_index) + return RebuildingECDiskFileStream( + metadata, rx_node_index, iter([content])) + + # create ssync sender instance... + job = {'device': self.device, + 'partition': self.partition, + 'policy': policy, + 'frag_index': frag_index, + 'sync_diskfile_builder': fake_reconstruct_fa} + node = {'index': rx_node_index} + self.sender = ssync_sender.Sender(self.daemon, node, job, suffixes) + + # fake connection from tx to rx... + self.sender.connect = self.make_fake_ssync_connect( + self.sender, self.rx_controller, self.device, self.partition, + policy) + + # run the sync protocol... + self.sender() + + # verify protocol + results = self._analyze_trace(self.sender.connection.trace) + # sender has primary for o1, o2 and o3, o4 and ts for o5 + self.assertEqual(5, len(results['tx_missing'])) + # receiver is missing o1, o2 and o3 and ts for o5 + self.assertEqual(4, len(results['rx_missing'])) + # sender can only construct 2 out of 3 missing frags + self.assertEqual(3, len(results['tx_updates'])) + self.assertEqual(3, len(reconstruct_fa_calls)) + self.assertFalse(results['rx_updates']) + actual_sync_paths = [] + for subreq in results.get('tx_updates'): + if subreq.get('method') == 'PUT': + self.assertTrue( + 'X-Object-Sysmeta-Ec-Frag-Index: %s' % rx_node_index + in subreq.get('headers')) + expected_body = '%s___%s' % (subreq['path'], rx_node_index) + self.assertEqual(expected_body, subreq['body']) + elif subreq.get('method') == 'DELETE': + self.assertEqual('/a/c/o5', subreq['path']) + actual_sync_paths.append(subreq.get('path')) + + # remove the failed df from expected synced df's + expect_sync_paths = ['/a/c/o1', '/a/c/o2', '/a/c/o3', '/a/c/o5'] + failed_path = reconstruct_fa_calls[1][3]['name'] + expect_sync_paths.remove(failed_path) + failed_obj = None + for obj, diskfiles in tx_objs.iteritems(): + if diskfiles[0]._name == failed_path: + failed_obj = obj + # sanity check + self.assertTrue(tx_objs.pop(failed_obj)) + + # verify on disk files... + self.assertEqual(sorted(expect_sync_paths), sorted(actual_sync_paths)) + self._verify_ondisk_files(tx_objs, policy, rx_node_index) + self._verify_tombstones(tx_tombstones, policy) + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index 1915a55d1..2ca396545 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -70,7 +70,7 @@ class TestObjectUpdater(unittest.TestCase): self.sda1 = os.path.join(self.devices_dir, 'sda1') os.mkdir(self.sda1) for policy in POLICIES: - os.mkdir(os.path.join(self.sda1, get_tmp_dir(int(policy)))) + os.mkdir(os.path.join(self.sda1, get_tmp_dir(policy))) self.logger = debug_logger() def tearDown(self): @@ -169,8 +169,8 @@ class TestObjectUpdater(unittest.TestCase): seen = set() class MockObjectUpdater(object_updater.ObjectUpdater): - def process_object_update(self, update_path, device, idx): - seen.add((update_path, idx)) + def process_object_update(self, update_path, device, policy): + seen.add((update_path, int(policy))) os.unlink(update_path) cu = MockObjectUpdater({ @@ -216,7 +216,7 @@ class TestObjectUpdater(unittest.TestCase): 'concurrency': '1', 'node_timeout': '15'}) cu.run_once() - async_dir = os.path.join(self.sda1, get_async_dir(0)) + async_dir = os.path.join(self.sda1, get_async_dir(POLICIES[0])) os.mkdir(async_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) @@ -253,7 +253,7 @@ class TestObjectUpdater(unittest.TestCase): 'concurrency': '1', 'node_timeout': '15'}, logger=self.logger) cu.run_once() - async_dir = os.path.join(self.sda1, get_async_dir(0)) + async_dir = os.path.join(self.sda1, get_async_dir(POLICIES[0])) os.mkdir(async_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) @@ -393,7 +393,7 @@ class TestObjectUpdater(unittest.TestCase): 'mount_check': 'false', 'swift_dir': self.testdir, } - async_dir = os.path.join(self.sda1, get_async_dir(policy.idx)) + async_dir = os.path.join(self.sda1, get_async_dir(policy)) os.mkdir(async_dir) account, container, obj = 'a', 'c', 'o' @@ -412,7 +412,7 @@ class TestObjectUpdater(unittest.TestCase): data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} dfmanager.pickle_async_update(self.sda1, account, container, obj, - data, ts.next(), policy.idx) + data, ts.next(), policy) request_log = [] @@ -428,7 +428,7 @@ class TestObjectUpdater(unittest.TestCase): ip, part, method, path, headers, qs, ssl = request_args self.assertEqual(method, op) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], - str(policy.idx)) + str(int(policy))) self.assertEqual(daemon.logger.get_increment_counts(), {'successes': 1, 'unlinks': 1, 'async_pendings': 1}) @@ -444,7 +444,7 @@ class TestObjectUpdater(unittest.TestCase): 'swift_dir': self.testdir, } daemon = object_updater.ObjectUpdater(conf, logger=self.logger) - async_dir = os.path.join(self.sda1, get_async_dir(policy.idx)) + async_dir = os.path.join(self.sda1, get_async_dir(policy)) os.mkdir(async_dir) # write an async @@ -456,12 +456,12 @@ class TestObjectUpdater(unittest.TestCase): 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-timestamp': ts.next(), - 'X-Backend-Storage-Policy-Index': policy.idx, + 'X-Backend-Storage-Policy-Index': int(policy), }) data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} dfmanager.pickle_async_update(self.sda1, account, container, obj, - data, ts.next(), policy.idx) + data, ts.next(), policy) request_log = [] @@ -481,7 +481,7 @@ class TestObjectUpdater(unittest.TestCase): ip, part, method, path, headers, qs, ssl = request_args self.assertEqual(method, 'PUT') self.assertEqual(headers['X-Backend-Storage-Policy-Index'], - str(policy.idx)) + str(int(policy))) self.assertEqual(daemon.logger.get_increment_counts(), {'successes': 1, 'unlinks': 1, 'async_pendings': 1}) diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 2c2094ffe..037e28b44 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -21,9 +21,11 @@ from swift.proxy.controllers.base import headers_to_container_info, \ headers_to_account_info, headers_to_object_info, get_container_info, \ get_container_memcache_key, get_account_info, get_account_memcache_key, \ get_object_env_key, get_info, get_object_info, \ - Controller, GetOrHeadHandler, _set_info_cache, _set_object_info_cache + Controller, GetOrHeadHandler, _set_info_cache, _set_object_info_cache, \ + bytes_to_skip from swift.common.swob import Request, HTTPException, HeaderKeyDict, \ RESPONSE_REASONS +from swift.common import exceptions from swift.common.utils import split_path from swift.common.http import is_success from swift.common.storage_policy import StoragePolicy @@ -159,9 +161,11 @@ class TestFuncs(unittest.TestCase): def test_GETorHEAD_base(self): base = Controller(self.app) req = Request.blank('/v1/a/c/o/with/slashes') + ring = FakeRing() + nodes = list(ring.get_part_nodes(0)) + list(ring.get_more_nodes(0)) with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', + resp = base.GETorHEAD_base(req, 'object', iter(nodes), 'part', '/a/c/o/with/slashes') self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ) self.assertEqual( @@ -169,14 +173,14 @@ class TestFuncs(unittest.TestCase): req = Request.blank('/v1/a/c/o') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', + resp = base.GETorHEAD_base(req, 'object', iter(nodes), 'part', '/a/c/o') self.assertTrue('swift.object/a/c/o' in resp.environ) self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200) req = Request.blank('/v1/a/c') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part', + resp = base.GETorHEAD_base(req, 'container', iter(nodes), 'part', '/a/c') self.assertTrue('swift.container/a/c' in resp.environ) self.assertEqual(resp.environ['swift.container/a/c']['status'], 200) @@ -184,7 +188,7 @@ class TestFuncs(unittest.TestCase): req = Request.blank('/v1/a') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part', + resp = base.GETorHEAD_base(req, 'account', iter(nodes), 'part', '/a') self.assertTrue('swift.account/a' in resp.environ) self.assertEqual(resp.environ['swift.account/a']['status'], 200) @@ -546,7 +550,7 @@ class TestFuncs(unittest.TestCase): resp, headers_to_object_info(headers.items(), 200)) - def test_have_quorum(self): + def test_base_have_quorum(self): base = Controller(self.app) # just throw a bunch of test cases at it self.assertEqual(base.have_quorum([201, 404], 3), False) @@ -648,3 +652,88 @@ class TestFuncs(unittest.TestCase): self.assertEqual(v, dst_headers[k.lower()]) for k, v in bad_hdrs.iteritems(): self.assertFalse(k.lower() in dst_headers) + + def test_client_chunk_size(self): + + class TestSource(object): + def __init__(self, chunks): + self.chunks = list(chunks) + + def read(self, _read_size): + if self.chunks: + return self.chunks.pop(0) + else: + return '' + + source = TestSource(( + 'abcd', '1234', 'abc', 'd1', '234abcd1234abcd1', '2')) + req = Request.blank('/v1/a/c/o') + node = {} + handler = GetOrHeadHandler(self.app, req, None, None, None, None, {}, + client_chunk_size=8) + + app_iter = handler._make_app_iter(req, node, source) + client_chunks = list(app_iter) + self.assertEqual(client_chunks, [ + 'abcd1234', 'abcd1234', 'abcd1234', 'abcd12']) + + def test_client_chunk_size_resuming(self): + + class TestSource(object): + def __init__(self, chunks): + self.chunks = list(chunks) + + def read(self, _read_size): + if self.chunks: + chunk = self.chunks.pop(0) + if chunk is None: + raise exceptions.ChunkReadTimeout() + else: + return chunk + else: + return '' + + node = {'ip': '1.2.3.4', 'port': 6000, 'device': 'sda'} + + source1 = TestSource(['abcd', '1234', 'abc', None]) + source2 = TestSource(['efgh5678']) + req = Request.blank('/v1/a/c/o') + handler = GetOrHeadHandler( + self.app, req, 'Object', None, None, None, {}, + client_chunk_size=8) + + app_iter = handler._make_app_iter(req, node, source1) + with patch.object(handler, '_get_source_and_node', + lambda: (source2, node)): + client_chunks = list(app_iter) + self.assertEqual(client_chunks, ['abcd1234', 'efgh5678']) + self.assertEqual(handler.backend_headers['Range'], 'bytes=8-') + + def test_bytes_to_skip(self): + # if you start at the beginning, skip nothing + self.assertEqual(bytes_to_skip(1024, 0), 0) + + # missed the first 10 bytes, so we've got 1014 bytes of partial + # record + self.assertEqual(bytes_to_skip(1024, 10), 1014) + + # skipped some whole records first + self.assertEqual(bytes_to_skip(1024, 4106), 1014) + + # landed on a record boundary + self.assertEqual(bytes_to_skip(1024, 1024), 0) + self.assertEqual(bytes_to_skip(1024, 2048), 0) + + # big numbers + self.assertEqual(bytes_to_skip(2 ** 20, 2 ** 32), 0) + self.assertEqual(bytes_to_skip(2 ** 20, 2 ** 32 + 1), 2 ** 20 - 1) + self.assertEqual(bytes_to_skip(2 ** 20, 2 ** 32 + 2 ** 19), 2 ** 19) + + # odd numbers + self.assertEqual(bytes_to_skip(123, 0), 0) + self.assertEqual(bytes_to_skip(123, 23), 100) + self.assertEqual(bytes_to_skip(123, 247), 122) + + # prime numbers + self.assertEqual(bytes_to_skip(11, 7), 4) + self.assertEqual(bytes_to_skip(97, 7873823), 55) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 002582a1a..a38e753ae 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -14,11 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import email.parser import itertools import random import time import unittest +from collections import defaultdict from contextlib import contextmanager +import json +from hashlib import md5 import mock from eventlet import Timeout @@ -26,13 +30,26 @@ from eventlet import Timeout import swift from swift.common import utils, swob from swift.proxy import server as proxy_server -from swift.common.storage_policy import StoragePolicy, POLICIES +from swift.proxy.controllers import obj +from swift.proxy.controllers.base import get_info as _real_get_info +from swift.common.storage_policy import POLICIES, ECDriverError from test.unit import FakeRing, FakeMemcache, fake_http_connect, \ - debug_logger, patch_policies + debug_logger, patch_policies, SlowBody from test.unit.proxy.test_server import node_error_count +def unchunk_body(chunked_body): + body = '' + remaining = chunked_body + while remaining: + hex_length, remaining = remaining.split('\r\n', 1) + length = int(hex_length, 16) + body += remaining[:length] + remaining = remaining[length + 2:] + return body + + @contextmanager def set_http_connect(*args, **kwargs): old_connect = swift.proxy.controllers.base.http_connect @@ -55,31 +72,76 @@ def set_http_connect(*args, **kwargs): class PatchedObjControllerApp(proxy_server.Application): """ - This patch is just a hook over handle_request to ensure that when - get_controller is called the ObjectController class is patched to - return a (possibly stubbed) ObjectController class. + This patch is just a hook over the proxy server's __call__ to ensure + that calls to get_info will return the stubbed value for + container_info if it's a container info call. """ - object_controller = proxy_server.ObjectController + container_info = {} + per_container_info = {} + + def __call__(self, *args, **kwargs): - def handle_request(self, req): - with mock.patch('swift.proxy.server.ObjectController', - new=self.object_controller): - return super(PatchedObjControllerApp, self).handle_request(req) + def _fake_get_info(app, env, account, container=None, **kwargs): + if container: + if container in self.per_container_info: + return self.per_container_info[container] + return self.container_info + else: + return _real_get_info(app, env, account, container, **kwargs) + mock_path = 'swift.proxy.controllers.base.get_info' + with mock.patch(mock_path, new=_fake_get_info): + return super( + PatchedObjControllerApp, self).__call__(*args, **kwargs) + + +class BaseObjectControllerMixin(object): + container_info = { + 'write_acl': None, + 'read_acl': None, + 'storage_policy': None, + 'sync_key': None, + 'versions': None, + } + + # this needs to be set on the test case + controller_cls = None -@patch_policies([StoragePolicy(0, 'zero', True, - object_ring=FakeRing(max_more_nodes=9))]) -class TestObjControllerWriteAffinity(unittest.TestCase): def setUp(self): - self.app = proxy_server.Application( + # setup fake rings with handoffs + for policy in POLICIES: + policy.object_ring.max_more_nodes = policy.object_ring.replicas + + self.logger = debug_logger('proxy-server') + self.logger.thread_locals = ('txn1', '127.0.0.2') + self.app = PatchedObjControllerApp( None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), logger=debug_logger()) - self.app.request_node_count = lambda ring: 10000000 - self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes + container_ring=FakeRing(), logger=self.logger) + # you can over-ride the container_info just by setting it on the app + self.app.container_info = dict(self.container_info) + # default policy and ring references + self.policy = POLICIES.default + self.obj_ring = self.policy.object_ring + self._ts_iter = (utils.Timestamp(t) for t in + itertools.count(int(time.time()))) + + def ts(self): + return self._ts_iter.next() + + def replicas(self, policy=None): + policy = policy or POLICIES.default + return policy.object_ring.replicas + + def quorum(self, policy=None): + policy = policy or POLICIES.default + return policy.quorum def test_iter_nodes_local_first_noops_when_no_affinity(self): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + # this test needs a stable node order - most don't + self.app.sort_nodes = lambda l: l + controller = self.controller_cls( + self.app, 'a', 'c', 'o') self.app.write_affinity_is_local_fn = None object_ring = self.app.get_object_ring(None) all_nodes = object_ring.get_part_nodes(1) @@ -93,80 +155,335 @@ class TestObjControllerWriteAffinity(unittest.TestCase): self.assertEqual(all_nodes, local_first_nodes) def test_iter_nodes_local_first_moves_locals_first(self): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = self.controller_cls( + self.app, 'a', 'c', 'o') self.app.write_affinity_is_local_fn = ( lambda node: node['region'] == 1) - self.app.write_affinity_node_count = lambda ring: 4 + # we'll write to one more than replica count local nodes + self.app.write_affinity_node_count = lambda r: r + 1 object_ring = self.app.get_object_ring(None) + # make our fake ring have plenty of nodes, and not get limited + # artificially by the proxy max request node count + object_ring.max_more_nodes = 100000 + self.app.request_node_count = lambda r: 100000 + all_nodes = object_ring.get_part_nodes(1) all_nodes.extend(object_ring.get_more_nodes(1)) + # i guess fake_ring wants the get_more_nodes iter to more safely be + # converted to a list with a smallish sort of limit which *can* be + # lower than max_more_nodes + fake_rings_real_max_more_nodes_value = object_ring.replicas ** 2 + self.assertEqual(len(all_nodes), fake_rings_real_max_more_nodes_value) + + # make sure we have enough local nodes (sanity) + all_local_nodes = [n for n in all_nodes if + self.app.write_affinity_is_local_fn(n)] + self.assertTrue(len(all_local_nodes) >= self.replicas() + 1) + + # finally, create the local_first_nodes iter and flatten it out local_first_nodes = list(controller.iter_nodes_local_first( object_ring, 1)) # the local nodes move up in the ordering - self.assertEqual([1, 1, 1, 1], - [node['region'] for node in local_first_nodes[:4]]) + self.assertEqual([1] * (self.replicas() + 1), [ + node['region'] for node in local_first_nodes[ + :self.replicas() + 1]]) # we don't skip any nodes self.assertEqual(len(all_nodes), len(local_first_nodes)) self.assertEqual(sorted(all_nodes), sorted(local_first_nodes)) + def test_iter_nodes_local_first_best_effort(self): + controller = self.controller_cls( + self.app, 'a', 'c', 'o') + self.app.write_affinity_is_local_fn = ( + lambda node: node['region'] == 1) + + object_ring = self.app.get_object_ring(None) + all_nodes = object_ring.get_part_nodes(1) + all_nodes.extend(object_ring.get_more_nodes(1)) + + local_first_nodes = list(controller.iter_nodes_local_first( + object_ring, 1)) + + # we won't have quite enough local nodes... + self.assertEqual(len(all_nodes), self.replicas() + + POLICIES.default.object_ring.max_more_nodes) + all_local_nodes = [n for n in all_nodes if + self.app.write_affinity_is_local_fn(n)] + self.assertEqual(len(all_local_nodes), self.replicas()) + # but the local nodes we do have are at the front of the local iter + first_n_local_first_nodes = local_first_nodes[:len(all_local_nodes)] + self.assertEqual(sorted(all_local_nodes), + sorted(first_n_local_first_nodes)) + # but we *still* don't *skip* any nodes + self.assertEqual(len(all_nodes), len(local_first_nodes)) + self.assertEqual(sorted(all_nodes), sorted(local_first_nodes)) + def test_connect_put_node_timeout(self): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = self.controller_cls( + self.app, 'a', 'c', 'o') self.app.conn_timeout = 0.05 with set_http_connect(slow_connect=True): nodes = [dict(ip='', port='', device='')] res = controller._connect_put_node(nodes, '', '', {}, ('', '')) self.assertTrue(res is None) + def test_DELETE_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + codes = [204] * self.replicas() + with set_http_connect(*codes): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 204) + + def test_DELETE_missing_one(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + codes = [404] + [204] * (self.replicas() - 1) + random.shuffle(codes) + with set_http_connect(*codes): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 204) -@patch_policies([ - StoragePolicy(0, 'zero', True), - StoragePolicy(1, 'one'), - StoragePolicy(2, 'two'), -]) -class TestObjController(unittest.TestCase): - container_info = { - 'partition': 1, - 'nodes': [ - {'ip': '127.0.0.1', 'port': '1', 'device': 'sda'}, - {'ip': '127.0.0.1', 'port': '2', 'device': 'sda'}, - {'ip': '127.0.0.1', 'port': '3', 'device': 'sda'}, - ], - 'write_acl': None, - 'read_acl': None, - 'storage_policy': None, - 'sync_key': None, - 'versions': None, - } + def test_DELETE_not_found(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + codes = [404] * (self.replicas() - 1) + [204] + with set_http_connect(*codes): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 404) - def setUp(self): - # setup fake rings with handoffs - self.obj_ring = FakeRing(max_more_nodes=3) - for policy in POLICIES: - policy.object_ring = self.obj_ring + def test_DELETE_mostly_found(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + mostly_204s = [204] * self.quorum() + codes = mostly_204s + [404] * (self.replicas() - len(mostly_204s)) + self.assertEqual(len(codes), self.replicas()) + with set_http_connect(*codes): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 204) - logger = debug_logger('proxy-server') - logger.thread_locals = ('txn1', '127.0.0.2') - self.app = PatchedObjControllerApp( - None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), logger=logger) + def test_DELETE_mostly_not_found(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + mostly_404s = [404] * self.quorum() + codes = mostly_404s + [204] * (self.replicas() - len(mostly_404s)) + self.assertEqual(len(codes), self.replicas()) + with set_http_connect(*codes): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 404) + + def test_DELETE_half_not_found_statuses(self): + self.obj_ring.set_replicas(4) + + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + with set_http_connect(404, 204, 404, 204): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 204) + + def test_DELETE_half_not_found_headers_and_body(self): + # Transformed responses have bogus bodies and headers, so make sure we + # send the client headers and body from a real node's response. + self.obj_ring.set_replicas(4) + + status_codes = (404, 404, 204, 204) + bodies = ('not found', 'not found', '', '') + headers = [{}, {}, {'Pick-Me': 'yes'}, {'Pick-Me': 'yes'}] + + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + with set_http_connect(*status_codes, body_iter=bodies, + headers=headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get('Pick-Me'), 'yes') + self.assertEquals(resp.body, '') + + def test_DELETE_handoff(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') + codes = [204] * self.replicas() + with set_http_connect(507, *codes): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 204) + + def test_POST_non_int_delete_after(self): + t = str(int(time.time() + 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': t}) + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-After', resp.body) + + def test_PUT_non_int_delete_after(self): + t = str(int(time.time() + 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': t}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-After', resp.body) + + def test_POST_negative_delete_after(self): + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': '-60'}) + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-After in past', resp.body) + + def test_PUT_negative_delete_after(self): + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': '-60'}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-After in past', resp.body) + + def test_POST_delete_at_non_integer(self): + t = str(int(time.time() + 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-At', resp.body) + + def test_PUT_delete_at_non_integer(self): + t = str(int(time.time() - 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-At', resp.body) + + def test_POST_delete_at_in_past(self): + t = str(int(time.time() - 100)) + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-At in past', resp.body) + + def test_PUT_delete_at_in_past(self): + t = str(int(time.time() - 100)) + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-At in past', resp.body) - class FakeContainerInfoObjController(proxy_server.ObjectController): + def test_HEAD_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD') + with set_http_connect(200): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 200) - def container_info(controller, *args, **kwargs): - patch_path = 'swift.proxy.controllers.base.get_info' - with mock.patch(patch_path) as mock_get_info: - mock_get_info.return_value = dict(self.container_info) - return super(FakeContainerInfoObjController, - controller).container_info(*args, **kwargs) + def test_HEAD_x_newest(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD', + headers={'X-Newest': 'true'}) + with set_http_connect(200, 200, 200): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 200) + + def test_HEAD_x_newest_different_timestamps(self): + req = swob.Request.blank('/v1/a/c/o', method='HEAD', + headers={'X-Newest': 'true'}) + ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) + timestamps = [next(ts) for i in range(3)] + newest_timestamp = timestamps[-1] + random.shuffle(timestamps) + backend_response_headers = [{ + 'X-Backend-Timestamp': t.internal, + 'X-Timestamp': t.normal + } for t in timestamps] + with set_http_connect(200, 200, 200, + headers=backend_response_headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['x-timestamp'], newest_timestamp.normal) - # this is taking advantage of the fact that self.app is a - # PachedObjControllerApp, so handle_response will route into an - # instance of our FakeContainerInfoObjController just by - # overriding the class attribute for object_controller - self.app.object_controller = FakeContainerInfoObjController + def test_HEAD_x_newest_with_two_vector_timestamps(self): + req = swob.Request.blank('/v1/a/c/o', method='HEAD', + headers={'X-Newest': 'true'}) + ts = (utils.Timestamp(time.time(), offset=offset) + for offset in itertools.count()) + timestamps = [next(ts) for i in range(3)] + newest_timestamp = timestamps[-1] + random.shuffle(timestamps) + backend_response_headers = [{ + 'X-Backend-Timestamp': t.internal, + 'X-Timestamp': t.normal + } for t in timestamps] + with set_http_connect(200, 200, 200, + headers=backend_response_headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['x-backend-timestamp'], + newest_timestamp.internal) + + def test_HEAD_x_newest_with_some_missing(self): + req = swob.Request.blank('/v1/a/c/o', method='HEAD', + headers={'X-Newest': 'true'}) + ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) + request_count = self.app.request_node_count(self.obj_ring.replicas) + backend_response_headers = [{ + 'x-timestamp': next(ts).normal, + } for i in range(request_count)] + responses = [404] * (request_count - 1) + responses.append(200) + request_log = [] + + def capture_requests(ip, port, device, part, method, path, + headers=None, **kwargs): + req = { + 'ip': ip, + 'port': port, + 'device': device, + 'part': part, + 'method': method, + 'path': path, + 'headers': headers, + } + request_log.append(req) + with set_http_connect(*responses, + headers=backend_response_headers, + give_connect=capture_requests): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + for req in request_log: + self.assertEqual(req['method'], 'HEAD') + self.assertEqual(req['path'], '/a/c/o') + + def test_container_sync_delete(self): + ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) + test_indexes = [None] + [int(p) for p in POLICIES] + for policy_index in test_indexes: + req = swob.Request.blank( + '/v1/a/c/o', method='DELETE', headers={ + 'X-Timestamp': ts.next().internal}) + codes = [409] * self.obj_ring.replicas + ts_iter = itertools.repeat(ts.next().internal) + with set_http_connect(*codes, timestamps=ts_iter): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 409) + + def test_PUT_requires_length(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT') + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 411) + +# end of BaseObjectControllerMixin + + +@patch_policies() +class TestReplicatedObjController(BaseObjectControllerMixin, + unittest.TestCase): + + controller_cls = obj.ReplicatedObjectController def test_PUT_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT') @@ -279,56 +596,6 @@ class TestObjController(unittest.TestCase): resp = req.get_response(self.app) self.assertEquals(resp.status_int, 404) - def test_DELETE_simple(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') - with set_http_connect(204, 204, 204): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 204) - - def test_DELETE_missing_one(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') - with set_http_connect(404, 204, 204): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 204) - - def test_DELETE_half_not_found_statuses(self): - self.obj_ring.set_replicas(4) - - req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') - with set_http_connect(404, 204, 404, 204): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 204) - - def test_DELETE_half_not_found_headers_and_body(self): - # Transformed responses have bogus bodies and headers, so make sure we - # send the client headers and body from a real node's response. - self.obj_ring.set_replicas(4) - - status_codes = (404, 404, 204, 204) - bodies = ('not found', 'not found', '', '') - headers = [{}, {}, {'Pick-Me': 'yes'}, {'Pick-Me': 'yes'}] - - req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') - with set_http_connect(*status_codes, body_iter=bodies, - headers=headers): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 204) - self.assertEquals(resp.headers.get('Pick-Me'), 'yes') - self.assertEquals(resp.body, '') - - def test_DELETE_not_found(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') - with set_http_connect(404, 404, 204): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 404) - - def test_DELETE_handoff(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE') - codes = [204] * self.obj_ring.replicas - with set_http_connect(507, *codes): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 204) - def test_POST_as_COPY_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o', method='POST') head_resp = [200] * self.obj_ring.replicas + \ @@ -364,45 +631,11 @@ class TestObjController(unittest.TestCase): self.assertTrue('X-Delete-At-Partition' in given_headers) self.assertTrue('X-Delete-At-Container' in given_headers) - def test_POST_non_int_delete_after(self): - t = str(int(time.time() + 100)) + '.1' - req = swob.Request.blank('/v1/a/c/o', method='POST', - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': t}) - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('Non-integer X-Delete-After', resp.body) - - def test_POST_negative_delete_after(self): - req = swob.Request.blank('/v1/a/c/o', method='POST', - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '-60'}) - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('X-Delete-After in past', resp.body) - - def test_POST_delete_at_non_integer(self): - t = str(int(time.time() + 100)) + '.1' - req = swob.Request.blank('/v1/a/c/o', method='POST', - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('Non-integer X-Delete-At', resp.body) - - def test_POST_delete_at_in_past(self): - t = str(int(time.time() - 100)) - req = swob.Request.blank('/v1/a/c/o', method='POST', - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('X-Delete-At in past', resp.body) - - def test_PUT_converts_delete_after_to_delete_at(self): + def test_PUT_delete_at(self): + t = str(int(time.time() + 100)) req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '60'}) + 'X-Delete-At': t}) put_headers = [] def capture_headers(ip, port, device, part, method, path, headers, @@ -410,44 +643,20 @@ class TestObjController(unittest.TestCase): if method == 'PUT': put_headers.append(headers) codes = [201] * self.obj_ring.replicas - t = time.time() with set_http_connect(*codes, give_connect=capture_headers): - with mock.patch('time.time', lambda: t): - resp = req.get_response(self.app) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) - expected_delete_at = str(int(t) + 60) for given_headers in put_headers: - self.assertEquals(given_headers.get('X-Delete-At'), - expected_delete_at) + self.assertEquals(given_headers.get('X-Delete-At'), t) self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) self.assertTrue('X-Delete-At-Container' in given_headers) - def test_PUT_non_int_delete_after(self): - t = str(int(time.time() + 100)) + '.1' - req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': t}) - with set_http_connect(): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('Non-integer X-Delete-After', resp.body) - - def test_PUT_negative_delete_after(self): - req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '-60'}) - with set_http_connect(): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('X-Delete-After in past', resp.body) - - def test_PUT_delete_at(self): - t = str(int(time.time() + 100)) + def test_PUT_converts_delete_after_to_delete_at(self): req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) + 'X-Delete-After': '60'}) put_headers = [] def capture_headers(ip, port, device, part, method, path, headers, @@ -455,40 +664,24 @@ class TestObjController(unittest.TestCase): if method == 'PUT': put_headers.append(headers) codes = [201] * self.obj_ring.replicas + t = time.time() with set_http_connect(*codes, give_connect=capture_headers): - resp = req.get_response(self.app) + with mock.patch('time.time', lambda: t): + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) + expected_delete_at = str(int(t) + 60) for given_headers in put_headers: - self.assertEquals(given_headers.get('X-Delete-At'), t) + self.assertEquals(given_headers.get('X-Delete-At'), + expected_delete_at) self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) self.assertTrue('X-Delete-At-Container' in given_headers) - def test_PUT_delete_at_non_integer(self): - t = str(int(time.time() - 100)) + '.1' - req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - with set_http_connect(): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('Non-integer X-Delete-At', resp.body) - - def test_PUT_delete_at_in_past(self): - t = str(int(time.time() - 100)) - req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - with set_http_connect(): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 400) - self.assertEqual('X-Delete-At in past', resp.body) - def test_container_sync_put_x_timestamp_not_found(self): test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: - self.container_info['storage_policy'] = policy_index + self.app.container_info['storage_policy'] = policy_index put_timestamp = utils.Timestamp(time.time()).normal req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ @@ -502,7 +695,7 @@ class TestObjController(unittest.TestCase): def test_container_sync_put_x_timestamp_match(self): test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: - self.container_info['storage_policy'] = policy_index + self.app.container_info['storage_policy'] = policy_index put_timestamp = utils.Timestamp(time.time()).normal req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ @@ -518,7 +711,7 @@ class TestObjController(unittest.TestCase): ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: - self.container_info['storage_policy'] = policy_index + self.app.container_info['storage_policy'] = policy_index req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, @@ -544,19 +737,6 @@ class TestObjController(unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) - def test_container_sync_delete(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) - test_indexes = [None] + [int(p) for p in POLICIES] - for policy_index in test_indexes: - req = swob.Request.blank( - '/v1/a/c/o', method='DELETE', headers={ - 'X-Timestamp': ts.next().internal}) - codes = [409] * self.obj_ring.replicas - ts_iter = itertools.repeat(ts.next().internal) - with set_http_connect(*codes, timestamps=ts_iter): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 409) - def test_put_x_timestamp_conflict(self): ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) req = swob.Request.blank( @@ -624,88 +804,6 @@ class TestObjController(unittest.TestCase): resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) - def test_HEAD_simple(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD') - with set_http_connect(200): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 200) - - def test_HEAD_x_newest(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD', - headers={'X-Newest': 'true'}) - with set_http_connect(200, 200, 200): - resp = req.get_response(self.app) - self.assertEquals(resp.status_int, 200) - - def test_HEAD_x_newest_different_timestamps(self): - req = swob.Request.blank('/v1/a/c/o', method='HEAD', - headers={'X-Newest': 'true'}) - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) - timestamps = [next(ts) for i in range(3)] - newest_timestamp = timestamps[-1] - random.shuffle(timestamps) - backend_response_headers = [{ - 'X-Backend-Timestamp': t.internal, - 'X-Timestamp': t.normal - } for t in timestamps] - with set_http_connect(200, 200, 200, - headers=backend_response_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['x-timestamp'], newest_timestamp.normal) - - def test_HEAD_x_newest_with_two_vector_timestamps(self): - req = swob.Request.blank('/v1/a/c/o', method='HEAD', - headers={'X-Newest': 'true'}) - ts = (utils.Timestamp(time.time(), offset=offset) - for offset in itertools.count()) - timestamps = [next(ts) for i in range(3)] - newest_timestamp = timestamps[-1] - random.shuffle(timestamps) - backend_response_headers = [{ - 'X-Backend-Timestamp': t.internal, - 'X-Timestamp': t.normal - } for t in timestamps] - with set_http_connect(200, 200, 200, - headers=backend_response_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['x-backend-timestamp'], - newest_timestamp.internal) - - def test_HEAD_x_newest_with_some_missing(self): - req = swob.Request.blank('/v1/a/c/o', method='HEAD', - headers={'X-Newest': 'true'}) - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) - request_count = self.app.request_node_count(self.obj_ring.replicas) - backend_response_headers = [{ - 'x-timestamp': next(ts).normal, - } for i in range(request_count)] - responses = [404] * (request_count - 1) - responses.append(200) - request_log = [] - - def capture_requests(ip, port, device, part, method, path, - headers=None, **kwargs): - req = { - 'ip': ip, - 'port': port, - 'device': device, - 'part': part, - 'method': method, - 'path': path, - 'headers': headers, - } - request_log.append(req) - with set_http_connect(*responses, - headers=backend_response_headers, - give_connect=capture_requests): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 200) - for req in request_log: - self.assertEqual(req['method'], 'HEAD') - self.assertEqual(req['path'], '/a/c/o') - def test_PUT_log_info(self): req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT') req.headers['x-copy-from'] = 'some/where' @@ -731,18 +829,15 @@ class TestObjController(unittest.TestCase): self.assertEquals(req.environ.get('swift.log_info'), None) -@patch_policies([ - StoragePolicy(0, 'zero', True), - StoragePolicy(1, 'one'), - StoragePolicy(2, 'two'), -]) -class TestObjControllerLegacyCache(TestObjController): +@patch_policies(legacy_only=True) +class TestObjControllerLegacyCache(TestReplicatedObjController): """ This test pretends like memcache returned a stored value that should resemble whatever "old" format. It catches KeyErrors you'd get if your code was expecting some new format during a rolling upgrade. """ + # in this case policy_index is missing container_info = { 'read_acl': None, 'write_acl': None, @@ -750,6 +845,567 @@ class TestObjControllerLegacyCache(TestObjController): 'versions': None, } + def test_invalid_storage_policy_cache(self): + self.app.container_info['storage_policy'] = 1 + for method in ('GET', 'HEAD', 'POST', 'PUT', 'COPY'): + req = swob.Request.blank('/v1/a/c/o', method=method) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 503) + + +@patch_policies(with_ec_default=True) +class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): + container_info = { + 'read_acl': None, + 'write_acl': None, + 'sync_key': None, + 'versions': None, + 'storage_policy': '0', + } + + controller_cls = obj.ECObjectController + + def test_determine_chunk_destinations(self): + class FakePutter(object): + def __init__(self, index): + self.node_index = index + + controller = self.controller_cls( + self.app, 'a', 'c', 'o') + + # create a dummy list of putters, check no handoffs + putters = [] + for index in range(0, 4): + putters.append(FakePutter(index)) + got = controller._determine_chunk_destinations(putters) + expected = {} + for i, p in enumerate(putters): + expected[p] = i + self.assertEquals(got, expected) + + # now lets make a handoff at the end + putters[3].node_index = None + got = controller._determine_chunk_destinations(putters) + self.assertEquals(got, expected) + putters[3].node_index = 3 + + # now lets make a handoff at the start + putters[0].node_index = None + got = controller._determine_chunk_destinations(putters) + self.assertEquals(got, expected) + putters[0].node_index = 0 + + # now lets make a handoff in the middle + putters[2].node_index = None + got = controller._determine_chunk_destinations(putters) + self.assertEquals(got, expected) + putters[2].node_index = 0 + + # now lets make all of them handoffs + for index in range(0, 4): + putters[index].node_index = None + got = controller._determine_chunk_destinations(putters) + self.assertEquals(got, expected) + + def test_GET_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + get_resp = [200] * self.policy.ec_ndata + with set_http_connect(*get_resp): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 200) + + def test_GET_simple_x_newest(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', + headers={'X-Newest': 'true'}) + codes = [200] * self.replicas() + codes += [404] * self.obj_ring.max_more_nodes + with set_http_connect(*codes): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 200) + + def test_GET_error(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + get_resp = [503] + [200] * self.policy.ec_ndata + with set_http_connect(*get_resp): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 200) + + def test_GET_with_body(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + # turn a real body into fragments + segment_size = self.policy.ec_segment_size + real_body = ('asdf' * segment_size)[:-10] + # split it up into chunks + chunks = [real_body[x:x + segment_size] + for x in range(0, len(real_body), segment_size)] + fragment_payloads = [] + for chunk in chunks: + fragments = self.policy.pyeclib_driver.encode(chunk) + if not fragments: + break + fragment_payloads.append(fragments) + # sanity + sanity_body = '' + for fragment_payload in fragment_payloads: + sanity_body += self.policy.pyeclib_driver.decode( + fragment_payload) + self.assertEqual(len(real_body), len(sanity_body)) + self.assertEqual(real_body, sanity_body) + + node_fragments = zip(*fragment_payloads) + self.assertEqual(len(node_fragments), self.replicas()) # sanity + responses = [(200, ''.join(node_fragments[i]), {}) + for i in range(POLICIES.default.ec_ndata)] + status_codes, body_iter, headers = zip(*responses) + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 200) + self.assertEqual(len(real_body), len(resp.body)) + self.assertEqual(real_body, resp.body) + + def test_PUT_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [201] * self.replicas() + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + + def test_PUT_with_explicit_commit_status(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [(100, 100, 201)] * self.replicas() + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + + def test_PUT_error(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [503] * self.replicas() + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 503) + + def test_PUT_mostly_success(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [201] * self.quorum() + codes += [503] * (self.replicas() - len(codes)) + random.shuffle(codes) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + + def test_PUT_error_commit(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [(100, 503, Exception('not used'))] * self.replicas() + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 503) + + def test_PUT_mostly_success_commit(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [201] * self.quorum() + codes += [(100, 503, Exception('not used'))] * ( + self.replicas() - len(codes)) + random.shuffle(codes) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + + def test_PUT_mostly_error_commit(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [(100, 503, Exception('not used'))] * self.quorum() + codes += [201] * (self.replicas() - len(codes)) + random.shuffle(codes) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 503) + + def test_PUT_commit_timeout(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [201] * (self.replicas() - 1) + codes.append((100, Timeout(), Exception('not used'))) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + + def test_PUT_commit_exception(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + codes = [201] * (self.replicas() - 1) + codes.append((100, Exception('kaboom!'), Exception('not used'))) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + + def test_PUT_with_body(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT') + segment_size = self.policy.ec_segment_size + test_body = ('asdf' * segment_size)[:-10] + etag = md5(test_body).hexdigest() + size = len(test_body) + req.body = test_body + codes = [201] * self.replicas() + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + + put_requests = defaultdict(lambda: {'boundary': None, 'chunks': []}) + + def capture_body(conn_id, chunk): + put_requests[conn_id]['chunks'].append(chunk) + + def capture_headers(ip, port, device, part, method, path, headers, + **kwargs): + conn_id = kwargs['connection_id'] + put_requests[conn_id]['boundary'] = headers[ + 'X-Backend-Obj-Multipart-Mime-Boundary'] + + with set_http_connect(*codes, expect_headers=expect_headers, + give_send=capture_body, + give_connect=capture_headers): + resp = req.get_response(self.app) + + self.assertEquals(resp.status_int, 201) + frag_archives = [] + for connection_id, info in put_requests.items(): + body = unchunk_body(''.join(info['chunks'])) + self.assertTrue(info['boundary'] is not None, + "didn't get boundary for conn %r" % ( + connection_id,)) + + # email.parser.FeedParser doesn't know how to take a multipart + # message and boundary together and parse it; it only knows how + # to take a string, parse the headers, and figure out the + # boundary on its own. + parser = email.parser.FeedParser() + parser.feed( + "Content-Type: multipart/nobodycares; boundary=%s\r\n\r\n" % + info['boundary']) + parser.feed(body) + message = parser.close() + + self.assertTrue(message.is_multipart()) # sanity check + mime_parts = message.get_payload() + self.assertEqual(len(mime_parts), 3) + obj_part, footer_part, commit_part = mime_parts + + # attach the body to frag_archives list + self.assertEqual(obj_part['X-Document'], 'object body') + frag_archives.append(obj_part.get_payload()) + + # validate some footer metadata + self.assertEqual(footer_part['X-Document'], 'object metadata') + footer_metadata = json.loads(footer_part.get_payload()) + self.assertTrue(footer_metadata) + expected = { + 'X-Object-Sysmeta-EC-Content-Length': str(size), + 'X-Backend-Container-Update-Override-Size': str(size), + 'X-Object-Sysmeta-EC-Etag': etag, + 'X-Backend-Container-Update-Override-Etag': etag, + 'X-Object-Sysmeta-EC-Segment-Size': str(segment_size), + } + for header, value in expected.items(): + self.assertEqual(footer_metadata[header], value) + + # sanity on commit message + self.assertEqual(commit_part['X-Document'], 'put commit') + + self.assertEqual(len(frag_archives), self.replicas()) + fragment_size = self.policy.fragment_size + node_payloads = [] + for fa in frag_archives: + payload = [fa[x:x + fragment_size] + for x in range(0, len(fa), fragment_size)] + node_payloads.append(payload) + fragment_payloads = zip(*node_payloads) + + expected_body = '' + for fragment_payload in fragment_payloads: + self.assertEqual(len(fragment_payload), self.replicas()) + if True: + fragment_payload = list(fragment_payload) + expected_body += self.policy.pyeclib_driver.decode( + fragment_payload) + + self.assertEqual(len(test_body), len(expected_body)) + self.assertEqual(test_body, expected_body) + + def test_PUT_old_obj_server(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + responses = [ + # one server will response 100-continue but not include the + # needful expect headers and the connection will be dropped + ((100, Exception('not used')), {}), + ] + [ + # and pleanty of successful responses too + (201, { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes', + }), + ] * self.replicas() + random.shuffle(responses) + if responses[-1][0] != 201: + # whoops, stupid random + responses = responses[1:] + [responses[0]] + codes, expect_headers = zip(*responses) + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + + def test_COPY_cross_policy_type_from_replicated(self): + self.app.per_container_info = { + 'c1': self.app.container_info.copy(), + 'c2': self.app.container_info.copy(), + } + # make c2 use replicated storage policy 1 + self.app.per_container_info['c2']['storage_policy'] = '1' + + # a put request with copy from source c2 + req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT', + body='', headers={ + 'X-Copy-From': 'c2/o'}) + + # c2 get + codes = [200] * self.replicas(POLICIES[1]) + codes += [404] * POLICIES[1].object_ring.max_more_nodes + # c1 put + codes += [201] * self.replicas() + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 201) + + def test_COPY_cross_policy_type_to_replicated(self): + self.app.per_container_info = { + 'c1': self.app.container_info.copy(), + 'c2': self.app.container_info.copy(), + } + # make c1 use replicated storage policy 1 + self.app.per_container_info['c1']['storage_policy'] = '1' + + # a put request with copy from source c2 + req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT', + body='', headers={ + 'X-Copy-From': 'c2/o'}) + + # c2 get + codes = [200] * self.replicas() + codes += [404] * self.obj_ring.max_more_nodes + headers = { + 'X-Object-Sysmeta-Ec-Content-Length': 0, + } + # c1 put + codes += [201] * self.replicas(POLICIES[1]) + with set_http_connect(*codes, headers=headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 201) + + def test_COPY_cross_policy_type_unknown(self): + self.app.per_container_info = { + 'c1': self.app.container_info.copy(), + 'c2': self.app.container_info.copy(), + } + # make c1 use some made up storage policy index + self.app.per_container_info['c1']['storage_policy'] = '13' + + # a COPY request of c2 with destination in c1 + req = swift.common.swob.Request.blank('/v1/a/c2/o', method='COPY', + body='', headers={ + 'Destination': 'c1/o'}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 503) + + def _make_ec_archive_bodies(self, test_body, policy=None): + policy = policy or self.policy + segment_size = policy.ec_segment_size + # split up the body into buffers + chunks = [test_body[x:x + segment_size] + for x in range(0, len(test_body), segment_size)] + # encode the buffers into fragment payloads + fragment_payloads = [] + for chunk in chunks: + fragments = self.policy.pyeclib_driver.encode(chunk) + if not fragments: + break + fragment_payloads.append(fragments) + + # join up the fragment payloads per node + ec_archive_bodies = [''.join(fragments) + for fragments in zip(*fragment_payloads)] + return ec_archive_bodies + + def test_GET_mismatched_fragment_archives(self): + segment_size = self.policy.ec_segment_size + test_data1 = ('test' * segment_size)[:-333] + # N.B. the object data *length* here is different + test_data2 = ('blah1' * segment_size)[:-333] + + etag1 = md5(test_data1).hexdigest() + etag2 = md5(test_data2).hexdigest() + + ec_archive_bodies1 = self._make_ec_archive_bodies(test_data1) + ec_archive_bodies2 = self._make_ec_archive_bodies(test_data2) + + headers1 = {'X-Object-Sysmeta-Ec-Etag': etag1} + # here we're going to *lie* and say the etag here matches + headers2 = {'X-Object-Sysmeta-Ec-Etag': etag1} + + responses1 = [(200, body, headers1) + for body in ec_archive_bodies1] + responses2 = [(200, body, headers2) + for body in ec_archive_bodies2] + + req = swob.Request.blank('/v1/a/c/o') + + # sanity check responses1 + responses = responses1[:self.policy.ec_ndata] + status_codes, body_iter, headers = zip(*responses) + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertEqual(md5(resp.body).hexdigest(), etag1) + + # sanity check responses2 + responses = responses2[:self.policy.ec_ndata] + status_codes, body_iter, headers = zip(*responses) + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertEqual(md5(resp.body).hexdigest(), etag2) + + # now mix the responses a bit + mix_index = random.randint(0, self.policy.ec_ndata - 1) + mixed_responses = responses1[:self.policy.ec_ndata] + mixed_responses[mix_index] = responses2[mix_index] + + status_codes, body_iter, headers = zip(*mixed_responses) + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + try: + resp.body + except ECDriverError: + pass + else: + self.fail('invalid ec fragment response body did not blow up!') + error_lines = self.logger.get_lines_for_level('error') + self.assertEqual(1, len(error_lines)) + msg = error_lines[0] + self.assertTrue('Error decoding fragments' in msg) + self.assertTrue('/a/c/o' in msg) + log_msg_args, log_msg_kwargs = self.logger.log_dict['error'][0] + self.assertEqual(log_msg_kwargs['exc_info'][0], ECDriverError) + + def test_GET_read_timeout(self): + segment_size = self.policy.ec_segment_size + test_data = ('test' * segment_size)[:-333] + etag = md5(test_data).hexdigest() + ec_archive_bodies = self._make_ec_archive_bodies(test_data) + headers = {'X-Object-Sysmeta-Ec-Etag': etag} + self.app.recoverable_node_timeout = 0.01 + responses = [(200, SlowBody(body, 0.1), headers) + for body in ec_archive_bodies] + + req = swob.Request.blank('/v1/a/c/o') + + status_codes, body_iter, headers = zip(*responses + [ + (404, '', {}) for i in range( + self.policy.object_ring.max_more_nodes)]) + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + # do this inside the fake http context manager, it'll try to + # resume but won't be able to give us all the right bytes + self.assertNotEqual(md5(resp.body).hexdigest(), etag) + error_lines = self.logger.get_lines_for_level('error') + self.assertEqual(self.replicas(), len(error_lines)) + nparity = self.policy.ec_nparity + for line in error_lines[:nparity]: + self.assertTrue('retrying' in line) + for line in error_lines[nparity:]: + self.assertTrue('ChunkReadTimeout (0.01s)' in line) + + def test_GET_read_timeout_resume(self): + segment_size = self.policy.ec_segment_size + test_data = ('test' * segment_size)[:-333] + etag = md5(test_data).hexdigest() + ec_archive_bodies = self._make_ec_archive_bodies(test_data) + headers = {'X-Object-Sysmeta-Ec-Etag': etag} + self.app.recoverable_node_timeout = 0.05 + # first one is slow + responses = [(200, SlowBody(ec_archive_bodies[0], 0.1), headers)] + # ... the rest are fine + responses += [(200, body, headers) + for body in ec_archive_bodies[1:]] + + req = swob.Request.blank('/v1/a/c/o') + + status_codes, body_iter, headers = zip( + *responses[:self.policy.ec_ndata + 1]) + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertTrue(md5(resp.body).hexdigest(), etag) + error_lines = self.logger.get_lines_for_level('error') + self.assertEqual(1, len(error_lines)) + self.assertTrue('retrying' in error_lines[0]) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/test_mem_server.py b/test/unit/proxy/test_mem_server.py index bc5b8794f..f8bc2e321 100644 --- a/test/unit/proxy/test_mem_server.py +++ b/test/unit/proxy/test_mem_server.py @@ -34,7 +34,22 @@ class TestProxyServer(test_server.TestProxyServer): class TestObjectController(test_server.TestObjectController): - pass + def test_PUT_no_etag_fallocate(self): + # mem server doesn't call fallocate(), believe it or not + pass + + # these tests all go looking in the filesystem + def test_policy_IO(self): + pass + + def test_PUT_ec(self): + pass + + def test_PUT_ec_multiple_segments(self): + pass + + def test_PUT_ec_fragment_archive_etag_mismatch(self): + pass class TestContainerController(test_server.TestContainerController): diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 41f0ea3c3..08d3b363e 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,10 +15,13 @@ # limitations under the License. import logging +import math import os +import pickle import sys import unittest -from contextlib import contextmanager, nested +from contextlib import closing, contextmanager, nested +from gzip import GzipFile from shutil import rmtree from StringIO import StringIO import gc @@ -25,6 +29,7 @@ import time from textwrap import dedent from urllib import quote from hashlib import md5 +from pyeclib.ec_iface import ECDriverError from tempfile import mkdtemp import weakref import operator @@ -35,13 +40,14 @@ import random import mock from eventlet import sleep, spawn, wsgi, listen, Timeout -from swift.common.utils import json +from swift.common.utils import hash_path, json, storage_directory, public from test.unit import ( connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing, FakeMemcache, debug_logger, patch_policies, write_fake_ring, mocked_http_conn) from swift.proxy import server as proxy_server +from swift.proxy.controllers.obj import ReplicatedObjectController from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server @@ -49,16 +55,18 @@ from swift.common.middleware import proxy_logging from swift.common.middleware.acl import parse_acl, format_acl from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist from swift.common import utils, constraints +from swift.common.ring import RingData from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.wsgi import monkey_patch_mimetools, loadapp from swift.proxy.controllers import base as proxy_base from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers +import swift.proxy.controllers.obj from swift.common.swob import Request, Response, HTTPUnauthorized, \ - HTTPException, HTTPForbidden + HTTPException, HTTPForbidden, HeaderKeyDict from swift.common import storage_policy -from swift.common.storage_policy import StoragePolicy, \ +from swift.common.storage_policy import StoragePolicy, ECStoragePolicy, \ StoragePolicyCollection, POLICIES from swift.common.request_helpers import get_sys_meta_prefix @@ -87,10 +95,9 @@ def do_setup(the_object_server): os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked') mkdirs(_testdir) rmtree(_testdir) - mkdirs(os.path.join(_testdir, 'sda1')) - mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) - mkdirs(os.path.join(_testdir, 'sdb1')) - mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) + for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1', + 'sdf1', 'sdg1', 'sdh1', 'sdi1'): + mkdirs(os.path.join(_testdir, drive, 'tmp')) conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', @@ -102,8 +109,10 @@ def do_setup(the_object_server): con2lis = listen(('localhost', 0)) obj1lis = listen(('localhost', 0)) obj2lis = listen(('localhost', 0)) + obj3lis = listen(('localhost', 0)) + objsocks = [obj1lis, obj2lis, obj3lis] _test_sockets = \ - (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) + (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') account_devs = [ {'port': acc1lis.getsockname()[1]}, @@ -119,27 +128,45 @@ def do_setup(the_object_server): storage_policy._POLICIES = StoragePolicyCollection([ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), - StoragePolicy(2, 'two', False)]) + StoragePolicy(2, 'two', False), + ECStoragePolicy(3, 'ec', ec_type='jerasure_rs_vand', + ec_ndata=2, ec_nparity=1, ec_segment_size=4096)]) obj_rings = { 0: ('sda1', 'sdb1'), 1: ('sdc1', 'sdd1'), 2: ('sde1', 'sdf1'), + # sdg1, sdh1, sdi1 taken by policy 3 (see below) } for policy_index, devices in obj_rings.items(): policy = POLICIES[policy_index] - dev1, dev2 = devices obj_ring_path = os.path.join(_testdir, policy.ring_name + '.ring.gz') obj_devs = [ - {'port': obj1lis.getsockname()[1], 'device': dev1}, - {'port': obj2lis.getsockname()[1], 'device': dev2}, - ] + {'port': objsock.getsockname()[1], 'device': dev} + for objsock, dev in zip(objsocks, devices)] write_fake_ring(obj_ring_path, *obj_devs) + + # write_fake_ring can't handle a 3-element ring, and the EC policy needs + # at least 3 devs to work with, so we do it manually + devs = [{'id': 0, 'zone': 0, 'device': 'sdg1', 'ip': '127.0.0.1', + 'port': obj1lis.getsockname()[1]}, + {'id': 1, 'zone': 0, 'device': 'sdh1', 'ip': '127.0.0.1', + 'port': obj2lis.getsockname()[1]}, + {'id': 2, 'zone': 0, 'device': 'sdi1', 'ip': '127.0.0.1', + 'port': obj3lis.getsockname()[1]}] + pol3_replica2part2dev_id = [[0, 1, 2, 0], + [1, 2, 0, 1], + [2, 0, 1, 2]] + obj3_ring_path = os.path.join(_testdir, POLICIES[3].ring_name + '.ring.gz') + part_shift = 30 + with closing(GzipFile(obj3_ring_path, 'wb')) as fh: + pickle.dump(RingData(pol3_replica2part2dev_id, devs, part_shift), fh) + prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone(), logger=debug_logger('proxy')) for policy in POLICIES: # make sure all the rings are loaded prosrv.get_object_ring(policy.idx) - # don't loose this one! + # don't lose this one! _test_POLICIES = storage_policy._POLICIES acc1srv = account_server.AccountController( conf, logger=debug_logger('acct1')) @@ -153,8 +180,10 @@ def do_setup(the_object_server): conf, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( conf, logger=debug_logger('obj2')) + obj3srv = the_object_server.ObjectController( + conf, logger=debug_logger('obj3')) _test_servers = \ - (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) + (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware(prosrv, conf, logger=prosrv.logger) @@ -165,8 +194,9 @@ def do_setup(the_object_server): con2spa = spawn(wsgi.server, con2lis, con2srv, nl) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) + obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl) _test_coros = \ - (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa) + (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa) # Create account ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a') @@ -280,6 +310,15 @@ def sortHeaderNames(headerNames): return ', '.join(headers) +def parse_headers_string(headers_str): + headers_dict = HeaderKeyDict() + for line in headers_str.split('\r\n'): + if ': ' in line: + header, value = line.split(': ', 1) + headers_dict[header] = value + return headers_dict + + def node_error_count(proxy_app, ring_node): # Reach into the proxy's internals to get the error count for a # particular node @@ -846,12 +885,12 @@ class TestProxyServer(unittest.TestCase): self.assertTrue(app.admin_key is None) def test_get_info_controller(self): - path = '/info' + req = Request.blank('/info') app = proxy_server.Application({}, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing()) - controller, path_parts = app.get_controller(path) + controller, path_parts = app.get_controller(req) self.assertTrue('version' in path_parts) self.assertTrue(path_parts['version'] is None) @@ -861,6 +900,65 @@ class TestProxyServer(unittest.TestCase): self.assertEqual(controller.__name__, 'InfoController') + def test_error_limit_methods(self): + logger = debug_logger('test') + app = proxy_server.Application({}, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(), + logger=logger) + node = app.container_ring.get_part_nodes(0)[0] + # error occurred + app.error_occurred(node, 'test msg') + self.assertTrue('test msg' in + logger.get_lines_for_level('error')[-1]) + self.assertEqual(1, node_error_count(app, node)) + + # exception occurred + try: + raise Exception('kaboom1!') + except Exception as e1: + app.exception_occurred(node, 'test1', 'test1 msg') + line = logger.get_lines_for_level('error')[-1] + self.assertTrue('test1 server' in line) + self.assertTrue('test1 msg' in line) + log_args, log_kwargs = logger.log_dict['error'][-1] + self.assertTrue(log_kwargs['exc_info']) + self.assertEqual(log_kwargs['exc_info'][1], e1) + self.assertEqual(2, node_error_count(app, node)) + + # warning exception occurred + try: + raise Exception('kaboom2!') + except Exception as e2: + app.exception_occurred(node, 'test2', 'test2 msg', + level=logging.WARNING) + line = logger.get_lines_for_level('warning')[-1] + self.assertTrue('test2 server' in line) + self.assertTrue('test2 msg' in line) + log_args, log_kwargs = logger.log_dict['warning'][-1] + self.assertTrue(log_kwargs['exc_info']) + self.assertEqual(log_kwargs['exc_info'][1], e2) + self.assertEqual(3, node_error_count(app, node)) + + # custom exception occurred + try: + raise Exception('kaboom3!') + except Exception as e3: + e3_info = sys.exc_info() + try: + raise Exception('kaboom4!') + except Exception: + pass + app.exception_occurred(node, 'test3', 'test3 msg', + level=logging.WARNING, exc_info=e3_info) + line = logger.get_lines_for_level('warning')[-1] + self.assertTrue('test3 server' in line) + self.assertTrue('test3 msg' in line) + log_args, log_kwargs = logger.log_dict['warning'][-1] + self.assertTrue(log_kwargs['exc_info']) + self.assertEqual(log_kwargs['exc_info'][1], e3) + self.assertEqual(4, node_error_count(app, node)) + @patch_policies([ StoragePolicy(0, 'zero', is_default=True), @@ -981,6 +1079,23 @@ class TestObjectController(unittest.TestCase): for policy in POLICIES: policy.object_ring = FakeRing(base_port=3000) + def put_container(self, policy_name, container_name): + # Note: only works if called with unpatched policies + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: 0\r\n' + 'X-Storage-Token: t\r\n' + 'X-Storage-Policy: %s\r\n' + '\r\n' % (container_name, policy_name)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' + self.assertEqual(headers[:len(exp)], exp) + def assert_status_map(self, method, statuses, expected, raise_exc=False): with save_globals(): kwargs = {} @@ -1014,20 +1129,14 @@ class TestObjectController(unittest.TestCase): @unpatch_policies def test_policy_IO(self): - if hasattr(_test_servers[-1], '_filesystem'): - # ironically, the _filesystem attribute on the object server means - # the in-memory diskfile is in use, so this test does not apply - return - - def check_file(policy_idx, cont, devs, check_val): - partition, nodes = prosrv.get_object_ring(policy_idx).get_nodes( - 'a', cont, 'o') + def check_file(policy, cont, devs, check_val): + partition, nodes = policy.object_ring.get_nodes('a', cont, 'o') conf = {'devices': _testdir, 'mount_check': 'false'} df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) for dev in devs: file = df_mgr.get_diskfile(dev, partition, 'a', cont, 'o', - policy_idx=policy_idx) + policy=policy) if check_val is True: file.open() @@ -1058,8 +1167,8 @@ class TestObjectController(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(res.body, obj) - check_file(0, 'c', ['sda1', 'sdb1'], True) - check_file(0, 'c', ['sdc1', 'sdd1', 'sde1', 'sdf1'], False) + check_file(POLICIES[0], 'c', ['sda1', 'sdb1'], True) + check_file(POLICIES[0], 'c', ['sdc1', 'sdd1', 'sde1', 'sdf1'], False) # check policy 1: put file on c1, read it back, check loc on disk sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -1084,8 +1193,8 @@ class TestObjectController(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(res.body, obj) - check_file(1, 'c1', ['sdc1', 'sdd1'], True) - check_file(1, 'c1', ['sda1', 'sdb1', 'sde1', 'sdf1'], False) + check_file(POLICIES[1], 'c1', ['sdc1', 'sdd1'], True) + check_file(POLICIES[1], 'c1', ['sda1', 'sdb1', 'sde1', 'sdf1'], False) # check policy 2: put file on c2, read it back, check loc on disk sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -1110,8 +1219,8 @@ class TestObjectController(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(res.body, obj) - check_file(2, 'c2', ['sde1', 'sdf1'], True) - check_file(2, 'c2', ['sda1', 'sdb1', 'sdc1', 'sdd1'], False) + check_file(POLICIES[2], 'c2', ['sde1', 'sdf1'], True) + check_file(POLICIES[2], 'c2', ['sda1', 'sdb1', 'sdc1', 'sdd1'], False) @unpatch_policies def test_policy_IO_override(self): @@ -1146,7 +1255,7 @@ class TestObjectController(unittest.TestCase): conf = {'devices': _testdir, 'mount_check': 'false'} df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) df = df_mgr.get_diskfile(node['device'], partition, 'a', - 'c1', 'wrong-o', policy_idx=2) + 'c1', 'wrong-o', policy=POLICIES[2]) with df.open(): contents = ''.join(df.reader()) self.assertEqual(contents, "hello") @@ -1178,7 +1287,7 @@ class TestObjectController(unittest.TestCase): self.assertEqual(res.status_int, 204) df = df_mgr.get_diskfile(node['device'], partition, 'a', - 'c1', 'wrong-o', policy_idx=2) + 'c1', 'wrong-o', policy=POLICIES[2]) try: df.open() except DiskFileNotExist as e: @@ -1215,6 +1324,619 @@ class TestObjectController(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(res.body, obj) + @unpatch_policies + def test_PUT_ec(self): + policy = POLICIES[3] + self.put_container("ec", "ec-con") + + obj = 'abCD' * 10 # small, so we don't get multiple EC stripes + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/o1 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: "%s"\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + ecd = policy.pyeclib_driver + expected_pieces = set(ecd.encode(obj)) + + # go to disk to make sure it's there and all erasure-coded + partition, nodes = policy.object_ring.get_nodes('a', 'ec-con', 'o1') + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + got_pieces = set() + got_indices = set() + got_durable = [] + for node_index, node in enumerate(nodes): + df = df_mgr.get_diskfile(node['device'], partition, + 'a', 'ec-con', 'o1', + policy=policy) + with df.open(): + meta = df.get_metadata() + contents = ''.join(df.reader()) + got_pieces.add(contents) + + # check presence for a .durable file for the timestamp + durable_file = os.path.join( + _testdir, node['device'], storage_directory( + diskfile.get_data_dir(policy), + partition, hash_path('a', 'ec-con', 'o1')), + utils.Timestamp(df.timestamp).internal + '.durable') + + if os.path.isfile(durable_file): + got_durable.append(True) + + lmeta = dict((k.lower(), v) for k, v in meta.items()) + got_indices.add( + lmeta['x-object-sysmeta-ec-frag-index']) + + self.assertEqual( + lmeta['x-object-sysmeta-ec-etag'], + md5(obj).hexdigest()) + self.assertEqual( + lmeta['x-object-sysmeta-ec-content-length'], + str(len(obj))) + self.assertEqual( + lmeta['x-object-sysmeta-ec-segment-size'], + '4096') + self.assertEqual( + lmeta['x-object-sysmeta-ec-scheme'], + 'jerasure_rs_vand 2+1') + self.assertEqual( + lmeta['etag'], + md5(contents).hexdigest()) + + self.assertEqual(expected_pieces, got_pieces) + self.assertEqual(set(('0', '1', '2')), got_indices) + + # verify at least 2 puts made it all the way to the end of 2nd + # phase, ie at least 2 .durable statuses were written + num_durable_puts = sum(d is True for d in got_durable) + self.assertTrue(num_durable_puts >= 2) + + @unpatch_policies + def test_PUT_ec_multiple_segments(self): + ec_policy = POLICIES[3] + self.put_container("ec", "ec-con") + + pyeclib_header_size = len(ec_policy.pyeclib_driver.encode("")[0]) + segment_size = ec_policy.ec_segment_size + + # Big enough to have multiple segments. Also a multiple of the + # segment size to get coverage of that path too. + obj = 'ABC' * segment_size + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/o2 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + # it's a 2+1 erasure code, so each fragment archive should be half + # the length of the object, plus three inline pyeclib metadata + # things (one per segment) + expected_length = (len(obj) / 2 + pyeclib_header_size * 3) + + partition, nodes = ec_policy.object_ring.get_nodes( + 'a', 'ec-con', 'o2') + + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + got_durable = [] + fragment_archives = [] + for node in nodes: + df = df_mgr.get_diskfile( + node['device'], partition, 'a', + 'ec-con', 'o2', policy=ec_policy) + with df.open(): + contents = ''.join(df.reader()) + fragment_archives.append(contents) + self.assertEqual(len(contents), expected_length) + + # check presence for a .durable file for the timestamp + durable_file = os.path.join( + _testdir, node['device'], storage_directory( + diskfile.get_data_dir(ec_policy), + partition, hash_path('a', 'ec-con', 'o2')), + utils.Timestamp(df.timestamp).internal + '.durable') + + if os.path.isfile(durable_file): + got_durable.append(True) + + # Verify that we can decode each individual fragment and that they + # are all the correct size + fragment_size = ec_policy.fragment_size + nfragments = int( + math.ceil(float(len(fragment_archives[0])) / fragment_size)) + + for fragment_index in range(nfragments): + fragment_start = fragment_index * fragment_size + fragment_end = (fragment_index + 1) * fragment_size + + try: + frags = [fa[fragment_start:fragment_end] + for fa in fragment_archives] + seg = ec_policy.pyeclib_driver.decode(frags) + except ECDriverError: + self.fail("Failed to decode fragments %d; this probably " + "means the fragments are not the sizes they " + "should be" % fragment_index) + + segment_start = fragment_index * segment_size + segment_end = (fragment_index + 1) * segment_size + + self.assertEqual(seg, obj[segment_start:segment_end]) + + # verify at least 2 puts made it all the way to the end of 2nd + # phase, ie at least 2 .durable statuses were written + num_durable_puts = sum(d is True for d in got_durable) + self.assertTrue(num_durable_puts >= 2) + + @unpatch_policies + def test_PUT_ec_object_etag_mismatch(self): + self.put_container("ec", "ec-con") + + obj = '90:6A:02:60:B1:08-96da3e706025537fc42464916427727e' + prolis = _test_sockets[0] + prosrv = _test_servers[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/o3 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: %s\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5('something else').hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 422' + self.assertEqual(headers[:len(exp)], exp) + + # nothing should have made it to disk on the object servers + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'o3') + conf = {'devices': _testdir, 'mount_check': 'false'} + + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'o3') + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + for node in nodes: + df = df_mgr.get_diskfile(node['device'], partition, + 'a', 'ec-con', 'o3', policy=POLICIES[3]) + self.assertRaises(DiskFileNotExist, df.open) + + @unpatch_policies + def test_PUT_ec_fragment_archive_etag_mismatch(self): + self.put_container("ec", "ec-con") + + # Cause a hash mismatch by feeding one particular MD5 hasher some + # extra data. The goal here is to get exactly one of the hashers in + # an object server. + countdown = [1] + + def busted_md5_constructor(initial_str=""): + hasher = md5(initial_str) + if countdown[0] == 0: + hasher.update('wrong') + countdown[0] -= 1 + return hasher + + obj = 'uvarovite-esurience-cerated-symphysic' + prolis = _test_sockets[0] + prosrv = _test_servers[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + with mock.patch('swift.obj.server.md5', busted_md5_constructor): + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/pimento HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: %s\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 503' # no quorum + self.assertEqual(headers[:len(exp)], exp) + + # 2/3 of the fragment archives should have landed on disk + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'pimento') + conf = {'devices': _testdir, 'mount_check': 'false'} + + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'pimento') + conf = {'devices': _testdir, 'mount_check': 'false'} + + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + found = 0 + for node in nodes: + df = df_mgr.get_diskfile(node['device'], partition, + 'a', 'ec-con', 'pimento', + policy=POLICIES[3]) + try: + df.open() + found += 1 + except DiskFileNotExist: + pass + self.assertEqual(found, 2) + + @unpatch_policies + def test_PUT_ec_if_none_match(self): + self.put_container("ec", "ec-con") + + obj = 'ananepionic-lepidophyllous-ropewalker-neglectful' + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/inm HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: "%s"\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/inm HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'If-None-Match: *\r\n' + 'Etag: "%s"\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 412' + self.assertEqual(headers[:len(exp)], exp) + + @unpatch_policies + def test_GET_ec(self): + self.put_container("ec", "ec-con") + + obj = '0123456' * 11 * 17 + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/go-get-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'X-Object-Meta-Color: chartreuse\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/go-get-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + self.assertEqual('chartreuse', headers['X-Object-Meta-Color']) + + gotten_obj = '' + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + self.assertEqual(gotten_obj, obj) + + @unpatch_policies + def test_conditional_GET_ec(self): + self.put_container("ec", "ec-con") + + obj = 'this object has an etag and is otherwise unimportant' + etag = md5(obj).hexdigest() + not_etag = md5(obj + "blahblah").hexdigest() + + prolis = _test_sockets[0] + prosrv = _test_servers[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/conditionals HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + for verb in ('GET', 'HEAD'): + # If-Match + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-Match': etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-Match': not_etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 412) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-Match': "*"}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + + # If-None-Match + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-None-Match': etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 304) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-None-Match': not_etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-None-Match': "*"}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 304) + + @unpatch_policies + def test_GET_ec_big(self): + self.put_container("ec", "ec-con") + + # our EC segment size is 4 KiB, so this is multiple (3) segments; + # we'll verify that with a sanity check + obj = 'a moose once bit my sister' * 400 + self.assertTrue( + len(obj) > POLICIES.get_by_name("ec").ec_segment_size * 2, + "object is too small for proper testing") + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/big-obj-get HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/big-obj-get HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + + gotten_obj = '' + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + # This may look like a redundant test, but when things fail, this + # has a useful failure message while the subsequent one spews piles + # of garbage and demolishes your terminal's scrollback buffer. + self.assertEqual(len(gotten_obj), len(obj)) + self.assertEqual(gotten_obj, obj) + + @unpatch_policies + def test_GET_ec_failure_handling(self): + self.put_container("ec", "ec-con") + + obj = 'look at this object; it is simply amazing ' * 500 + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/crash-test-dummy HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + def explodey_iter(inner_iter): + yield next(inner_iter) + raise Exception("doom ba doom") + + real_ec_app_iter = swift.proxy.controllers.obj.ECAppIter + + def explodey_ec_app_iter(path, policy, iterators, *a, **kw): + # Each thing in `iterators` here is a document-parts iterator, + # and we want to fail after getting a little into each part. + # + # That way, we ensure we've started streaming the response to + # the client when things go wrong. + return real_ec_app_iter( + path, policy, + [explodey_iter(i) for i in iterators], + *a, **kw) + + with mock.patch("swift.proxy.controllers.obj.ECAppIter", + explodey_ec_app_iter): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/crash-test-dummy HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + + gotten_obj = '' + try: + with Timeout(300): # don't hang the testrun when this fails + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + except Timeout: + self.fail("GET hung when connection failed") + + # Ensure we failed partway through, otherwise the mocks could + # get out of date without anyone noticing + self.assertTrue(0 < len(gotten_obj) < len(obj)) + + @unpatch_policies + def test_HEAD_ec(self): + self.put_container("ec", "ec-con") + + obj = '0123456' * 11 * 17 + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/go-head-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'X-Object-Meta-Color: chartreuse\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('HEAD /v1/a/ec-con/go-head-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + self.assertEqual('chartreuse', headers['X-Object-Meta-Color']) + + @unpatch_policies + def test_GET_ec_404(self): + self.put_container("ec", "ec-con") + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/yes-we-have-no-bananas HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 404' + self.assertEqual(headers[:len(exp)], exp) + + @unpatch_policies + def test_HEAD_ec_404(self): + self.put_container("ec", "ec-con") + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('HEAD /v1/a/ec-con/yes-we-have-no-bananas HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 404' + self.assertEqual(headers[:len(exp)], exp) + def test_PUT_expect_header_zero_content_length(self): test_errors = [] @@ -1226,8 +1948,8 @@ class TestObjectController(unittest.TestCase): 'server!') with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') # The (201, Exception('test')) tuples in there have the effect of # changing the status of the initial expect response. The default # expect response from FakeConn for 201 is 100. @@ -1262,8 +1984,8 @@ class TestObjectController(unittest.TestCase): 'non-zero byte PUT!') with save_globals(): - controller = \ - proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') # the (100, 201) tuples in there are just being extra explicit # about the FakeConn returning the 100 Continue status when the # object controller calls getexpect. Which is FakeConn's default @@ -1298,7 +2020,8 @@ class TestObjectController(unittest.TestCase): self.app.write_affinity_node_count = lambda r: 3 controller = \ - proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') + ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') set_http_connect(200, 200, 201, 201, 201, give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) @@ -1333,7 +2056,8 @@ class TestObjectController(unittest.TestCase): self.app.write_affinity_node_count = lambda r: 3 controller = \ - proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') + ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') self.app.error_limit( object_ring.get_part_nodes(1)[0], 'test') set_http_connect(200, 200, # account, container @@ -1355,6 +2079,27 @@ class TestObjectController(unittest.TestCase): self.assertNotEqual(0, written_to[2][1] % 2) @unpatch_policies + def test_PUT_no_etag_fallocate(self): + with mock.patch('swift.obj.diskfile.fallocate') as mock_fallocate: + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + obj = 'hemoleucocytic-surfactant' + fd.write('PUT /v1/a/c/o HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + # one for each obj server; this test has 2 + self.assertEqual(len(mock_fallocate.mock_calls), 2) + + @unpatch_policies def test_PUT_message_length_using_content_length(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -1593,7 +2338,8 @@ class TestObjectController(unittest.TestCase): "last_modified": "1970-01-01T00:00:01.000000"}]) body_iter = ('', '', body, '', '', '', '', '', '', '', '', '', '', '') with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') # HEAD HEAD GET GET HEAD GET GET GET PUT PUT # PUT DEL DEL DEL set_http_connect(200, 200, 200, 200, 200, 200, 200, 200, 201, 201, @@ -1614,6 +2360,8 @@ class TestObjectController(unittest.TestCase): StoragePolicy(1, 'one', True, object_ring=FakeRing()) ]) def test_DELETE_on_expired_versioned_object(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() methods = set() authorize_call_count = [0] @@ -1646,8 +2394,8 @@ class TestObjectController(unittest.TestCase): return None # allow the request with save_globals(): - controller = proxy_server.ObjectController(self.app, - 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') controller.container_info = fake_container_info controller._listing_iter = fake_list_iter set_http_connect(404, 404, 404, # get for the previous version @@ -1678,6 +2426,8 @@ class TestObjectController(unittest.TestCase): Verify that a request with read access to a versions container is unable to cause any write operations on the versioned container. """ + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() methods = set() authorize_call_count = [0] @@ -1711,8 +2461,7 @@ class TestObjectController(unittest.TestCase): return HTTPForbidden(req) # allow the request with save_globals(): - controller = proxy_server.ObjectController(self.app, - 'a', 'c', 'o') + controller = ReplicatedObjectController(self.app, 'a', 'c', 'o') controller.container_info = fake_container_info # patching _listing_iter simulates request being authorized # to list versions container @@ -1731,8 +2480,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_auto_content_type(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_content_type(filename, expected): # The three responses here are for account_info() (HEAD to @@ -1778,8 +2527,8 @@ class TestObjectController(unittest.TestCase): def test_PUT(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): set_http_connect(*statuses) @@ -1798,8 +2547,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_connect_exceptions(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): set_http_connect(*statuses) @@ -1829,8 +2578,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_send_exceptions(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): self.app.memcache.store = {} @@ -1852,8 +2601,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_max_size(self): with save_globals(): set_http_connect(201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {}, headers={ 'Content-Length': str(constraints.MAX_FILE_SIZE + 1), 'Content-Type': 'foo/bar'}) @@ -1864,8 +2613,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_bad_content_type(self): with save_globals(): set_http_connect(201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {}, headers={ 'Content-Length': 0, 'Content-Type': 'foo/bar;swift_hey=45'}) self.app.update_request(req) @@ -1875,8 +2624,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_getresponse_exceptions(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): self.app.memcache.store = {} @@ -1921,6 +2670,8 @@ class TestObjectController(unittest.TestCase): StoragePolicy(1, 'one', object_ring=FakeRing()), ]) def test_POST_backend_headers(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() self.app.object_post_as_copy = False self.app.sort_nodes = lambda nodes: nodes backend_requests = [] @@ -2191,8 +2942,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): limit = constraints.MAX_META_VALUE_LENGTH self.app.object_post_as_copy = False - proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 202, 202, 202) # acct cont obj obj obj req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, @@ -2739,8 +3490,8 @@ class TestObjectController(unittest.TestCase): self.assertEqual(node_list, got_nodes) def test_best_response_sets_headers(self): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, 'Object', headers=[{'X-Test': '1'}, @@ -2749,8 +3500,8 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.headers['X-Test'], '1') def test_best_response_sets_etag(self): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, 'Object') @@ -2783,8 +3534,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 200) @@ -2796,8 +3547,8 @@ class TestObjectController(unittest.TestCase): def test_error_limiting(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') controller.app.sort_nodes = lambda l: l object_ring = controller.app.get_object_ring(None) self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200), @@ -2833,8 +3584,8 @@ class TestObjectController(unittest.TestCase): def test_error_limiting_survives_ring_reload(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') controller.app.sort_nodes = lambda l: l object_ring = controller.app.get_object_ring(None) self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200), @@ -2861,8 +3612,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_error_limiting(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') controller.app.sort_nodes = lambda l: l object_ring = controller.app.get_object_ring(None) # acc con obj obj obj @@ -2880,8 +3631,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_error_limiting_last_node(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') controller.app.sort_nodes = lambda l: l object_ring = controller.app.get_object_ring(None) # acc con obj obj obj @@ -2901,8 +3652,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() self.app._error_limiting = {} - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) @@ -2998,8 +3749,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): self.app.object_post_as_copy = False self.app.memcache = FakeMemcacheReturnsNone() - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 404, 404, 404, 200, 200, 200) req = Request.blank('/v1/a/c/o', @@ -3019,8 +3770,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_POST_as_copy_requires_container_exist(self): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 404, 404, 404, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) self.app.update_request(req) @@ -3037,8 +3788,8 @@ class TestObjectController(unittest.TestCase): def test_bad_metadata(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -3134,8 +3885,8 @@ class TestObjectController(unittest.TestCase): @contextmanager def controller_context(self, req, *args, **kwargs): _v, account, container, obj = utils.split_path(req.path, 4, 4, True) - controller = proxy_server.ObjectController(self.app, account, - container, obj) + controller = ReplicatedObjectController( + self.app, account, container, obj) self.app.update_request(req) self.app.memcache.store = {} with save_globals(): @@ -3752,7 +4503,8 @@ class TestObjectController(unittest.TestCase): def test_COPY_newest(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) @@ -3770,7 +4522,8 @@ class TestObjectController(unittest.TestCase): def test_COPY_account_newest(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c1/o', @@ -3795,8 +4548,8 @@ class TestObjectController(unittest.TestCase): headers=None, query_string=None): backend_requests.append((method, path, headers)) - controller = proxy_server.ObjectController(self.app, 'a', - 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, give_connect=capture_requests) self.app.memcache.store = {} @@ -3825,8 +4578,8 @@ class TestObjectController(unittest.TestCase): headers=None, query_string=None): backend_requests.append((method, path, headers)) - controller = proxy_server.ObjectController(self.app, 'a', - 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201, give_connect=capture_requests) self.app.memcache.store = {} @@ -3871,8 +4624,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): set_http_connect(201, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Transfer-Encoding': 'chunked', @@ -3902,7 +4655,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put_bad_version(self): # Check bad version (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n' @@ -3916,7 +4669,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put_bad_path(self): # Check bad path (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n' @@ -3930,7 +4683,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put_bad_utf8(self): # Check invalid utf-8 (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n' @@ -3945,7 +4698,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put_bad_path_no_controller(self): # Check bad path, no controller (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n' @@ -3960,7 +4713,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put_bad_method(self): # Check bad method (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n' @@ -3975,9 +4728,9 @@ class TestObjectController(unittest.TestCase): def test_chunked_put_unhandled_exception(self): # Check unhandled exception (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, - obj2srv) = _test_servers + obj2srv, obj3srv) = _test_servers (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets orig_update_request = prosrv.update_request def broken_update_request(*args, **kwargs): @@ -4001,7 +4754,7 @@ class TestObjectController(unittest.TestCase): # the part Application.log_request that 'enforces' a # content_length on the response. (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n' @@ -4025,7 +4778,7 @@ class TestObjectController(unittest.TestCase): ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest' # Create ustr container (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' @@ -4137,7 +4890,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put_chunked_put(self): # Do chunked object put (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() # Also happens to assert that x-storage-token is taken as a @@ -4168,7 +4921,7 @@ class TestObjectController(unittest.TestCase): versions_to_create = 3 # Create a container for our versioned object testing (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() pre = quote('%03x' % len(o)) @@ -4552,8 +5305,8 @@ class TestObjectController(unittest.TestCase): @unpatch_policies def test_conditional_range_get(self): - (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ - _test_sockets + (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, + obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) # make a container @@ -4601,8 +5354,8 @@ class TestObjectController(unittest.TestCase): def test_mismatched_etags(self): with save_globals(): # no etag supplied, object servers return success w/ diff values - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) self.app.update_request(req) @@ -4633,8 +5386,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.GET(req) self.assert_('accept-ranges' in resp.headers) @@ -4645,8 +5398,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.HEAD(req) self.assert_('accept-ranges' in resp.headers) @@ -4660,8 +5413,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o') req.environ['swift.authorize'] = authorize self.app.update_request(req) @@ -4676,8 +5429,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) req.environ['swift.authorize'] = authorize self.app.update_request(req) @@ -4693,8 +5446,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): self.app.object_post_as_copy = False set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Length': '5'}, body='12345') @@ -4711,8 +5464,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Length': '5'}, body='12345') @@ -4729,8 +5482,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') req.environ['swift.authorize'] = authorize @@ -4746,8 +5499,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c/o'}) @@ -4759,8 +5512,8 @@ class TestObjectController(unittest.TestCase): def test_POST_converts_delete_after_to_delete_at(self): with save_globals(): self.app.object_post_as_copy = False - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 202, 202, 202) self.app.memcache.store = {} orig_time = time.time @@ -4783,6 +5536,8 @@ class TestObjectController(unittest.TestCase): StoragePolicy(1, 'one', True, object_ring=FakeRing()) ]) def test_PUT_versioning_with_nonzero_default_policy(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): @@ -4808,8 +5563,8 @@ class TestObjectController(unittest.TestCase): {'zone': 2, 'ip': '10.0.0.2', 'region': 0, 'id': 2, 'device': 'sdc', 'port': 1002}]} with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', - 'c', 'o.jpg') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') controller.container_info = fake_container_info set_http_connect(200, 200, 200, # head: for the last version @@ -4830,6 +5585,8 @@ class TestObjectController(unittest.TestCase): StoragePolicy(1, 'one', True, object_ring=FakeRing()) ]) def test_cross_policy_DELETE_versioning(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() requests = [] def capture_requests(ipaddr, port, device, partition, method, path, @@ -4959,8 +5716,8 @@ class TestObjectController(unittest.TestCase): def test_OPTIONS(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', - 'c', 'o.jpg') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') def my_empty_container_info(*args): return {} @@ -5067,7 +5824,8 @@ class TestObjectController(unittest.TestCase): def test_CORS_valid(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') def stubContainerInfo(*args): return { @@ -5120,7 +5878,8 @@ class TestObjectController(unittest.TestCase): def test_CORS_valid_with_obj_headers(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') def stubContainerInfo(*args): return { @@ -5181,7 +5940,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_x_container_headers_with_equal_replicas(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT @@ -5202,7 +5962,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT @@ -5224,7 +5985,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT @@ -5248,7 +6010,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'application/stuff'}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.POST, req, 200, 200, 200, 200, 200) # HEAD HEAD POST POST POST @@ -5271,7 +6034,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'Content-Type': 'application/stuff'}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.DELETE, req, 200, 200, 200, 200, 200) # HEAD HEAD DELETE DELETE DELETE @@ -5300,7 +6064,8 @@ class TestObjectController(unittest.TestCase): headers={'Content-Type': 'application/stuff', 'Content-Length': '0', 'X-Delete-At': str(delete_at_timestamp)}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT @@ -5336,7 +6101,8 @@ class TestObjectController(unittest.TestCase): headers={'Content-Type': 'application/stuff', 'Content-Length': 0, 'X-Delete-At': str(delete_at_timestamp)}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT @@ -5358,6 +6124,373 @@ class TestObjectController(unittest.TestCase): ]) +class TestECMismatchedFA(unittest.TestCase): + def tearDown(self): + prosrv = _test_servers[0] + # don't leak error limits and poison other tests + prosrv._error_limiting = {} + + def test_mixing_different_objects_fragment_archives(self): + (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, + obj2srv, obj3srv) = _test_servers + ec_policy = POLICIES[3] + + @public + def bad_disk(req): + return Response(status=507, body="borken") + + ensure_container = Request.blank( + "/v1/a/ec-crazytown", + environ={"REQUEST_METHOD": "PUT"}, + headers={"X-Storage-Policy": "ec", "X-Auth-Token": "t"}) + resp = ensure_container.get_response(prosrv) + self.assertTrue(resp.status_int in (201, 202)) + + obj1 = "first version..." + put_req1 = Request.blank( + "/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "PUT"}, + headers={"X-Auth-Token": "t"}) + put_req1.body = obj1 + + obj2 = u"versión segundo".encode("utf-8") + put_req2 = Request.blank( + "/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "PUT"}, + headers={"X-Auth-Token": "t"}) + put_req2.body = obj2 + + # pyeclib has checks for unequal-length; we don't want to trip those + self.assertEqual(len(obj1), len(obj2)) + + # Servers obj1 and obj2 will have the first version of the object + prosrv._error_limiting = {} + with nested( + mock.patch.object(obj3srv, 'PUT', bad_disk), + mock.patch( + 'swift.common.storage_policy.ECStoragePolicy.quorum')): + type(ec_policy).quorum = mock.PropertyMock(return_value=2) + resp = put_req1.get_response(prosrv) + self.assertEqual(resp.status_int, 201) + + # Server obj3 (and, in real life, some handoffs) will have the + # second version of the object. + prosrv._error_limiting = {} + with nested( + mock.patch.object(obj1srv, 'PUT', bad_disk), + mock.patch.object(obj2srv, 'PUT', bad_disk), + mock.patch( + 'swift.common.storage_policy.ECStoragePolicy.quorum'), + mock.patch( + 'swift.proxy.controllers.base.Controller._quorum_size', + lambda *a, **kw: 1)): + type(ec_policy).quorum = mock.PropertyMock(return_value=1) + resp = put_req2.get_response(prosrv) + self.assertEqual(resp.status_int, 201) + + # A GET that only sees 1 fragment archive should fail + get_req = Request.blank("/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "GET"}, + headers={"X-Auth-Token": "t"}) + prosrv._error_limiting = {} + with nested( + mock.patch.object(obj1srv, 'GET', bad_disk), + mock.patch.object(obj2srv, 'GET', bad_disk)): + resp = get_req.get_response(prosrv) + self.assertEqual(resp.status_int, 503) + + # A GET that sees 2 matching FAs will work + get_req = Request.blank("/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "GET"}, + headers={"X-Auth-Token": "t"}) + prosrv._error_limiting = {} + with mock.patch.object(obj3srv, 'GET', bad_disk): + resp = get_req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.body, obj1) + + # A GET that sees 2 mismatching FAs will fail + get_req = Request.blank("/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "GET"}, + headers={"X-Auth-Token": "t"}) + prosrv._error_limiting = {} + with mock.patch.object(obj2srv, 'GET', bad_disk): + resp = get_req.get_response(prosrv) + self.assertEqual(resp.status_int, 503) + + +class TestObjectECRangedGET(unittest.TestCase): + def setUp(self): + self.app = proxy_server.Application( + None, FakeMemcache(), + logger=debug_logger('proxy-ut'), + account_ring=FakeRing(), + container_ring=FakeRing()) + + @classmethod + def setUpClass(cls): + cls.obj_name = 'range-get-test' + cls.tiny_obj_name = 'range-get-test-tiny' + cls.aligned_obj_name = 'range-get-test-aligned' + + # Note: only works if called with unpatched policies + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: 0\r\n' + 'X-Storage-Token: t\r\n' + 'X-Storage-Policy: ec\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' + assert headers[:len(exp)] == exp, "container PUT failed" + + seg_size = POLICIES.get_by_name("ec").ec_segment_size + cls.seg_size = seg_size + # EC segment size is 4 KiB, hence this gives 4 segments, which we + # then verify with a quick sanity check + cls.obj = ' my hovercraft is full of eels '.join( + str(s) for s in range(431)) + assert seg_size * 4 > len(cls.obj) > seg_size * 3, \ + "object is wrong number of segments" + + cls.tiny_obj = 'tiny, tiny object' + assert len(cls.tiny_obj) < seg_size, "tiny_obj too large" + + cls.aligned_obj = "".join( + "abcdEFGHijkl%04d" % x for x in range(512)) + assert len(cls.aligned_obj) % seg_size == 0, "aligned obj not aligned" + + for obj_name, obj in ((cls.obj_name, cls.obj), + (cls.tiny_obj_name, cls.tiny_obj), + (cls.aligned_obj_name, cls.aligned_obj)): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/%s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (obj_name, len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + assert headers[:len(exp)] == exp, \ + "object PUT failed %s" % obj_name + + def _get_obj(self, range_value, obj_name=None): + if obj_name is None: + obj_name = self.obj_name + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/%s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Range: %s\r\n' + '\r\n' % (obj_name, range_value)) + fd.flush() + headers = readuntil2crlfs(fd) + # e.g. "HTTP/1.1 206 Partial Content\r\n..." + status_code = int(headers[9:12]) + headers = parse_headers_string(headers) + + gotten_obj = '' + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + + return (status_code, headers, gotten_obj) + + def test_unaligned(self): + # One segment's worth of data, but straddling two segment boundaries + # (so it has data from three segments) + status, headers, gotten_obj = self._get_obj("bytes=3783-7878") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "4096") + self.assertEqual(headers['Content-Range'], "bytes 3783-7878/14513") + self.assertEqual(len(gotten_obj), 4096) + self.assertEqual(gotten_obj, self.obj[3783:7879]) + + def test_aligned_left(self): + # First byte is aligned to a segment boundary, last byte is not + status, headers, gotten_obj = self._get_obj("bytes=0-5500") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "5501") + self.assertEqual(headers['Content-Range'], "bytes 0-5500/14513") + self.assertEqual(len(gotten_obj), 5501) + self.assertEqual(gotten_obj, self.obj[:5501]) + + def test_aligned_range(self): + # Ranged GET that wants exactly one segment + status, headers, gotten_obj = self._get_obj("bytes=4096-8191") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "4096") + self.assertEqual(headers['Content-Range'], "bytes 4096-8191/14513") + self.assertEqual(len(gotten_obj), 4096) + self.assertEqual(gotten_obj, self.obj[4096:8192]) + + def test_aligned_range_end(self): + # Ranged GET that wants exactly the last segment + status, headers, gotten_obj = self._get_obj("bytes=12288-14512") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "2225") + self.assertEqual(headers['Content-Range'], "bytes 12288-14512/14513") + self.assertEqual(len(gotten_obj), 2225) + self.assertEqual(gotten_obj, self.obj[12288:]) + + def test_aligned_range_aligned_obj(self): + # Ranged GET that wants exactly the last segment, which is full-size + status, headers, gotten_obj = self._get_obj("bytes=4096-8191", + self.aligned_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "4096") + self.assertEqual(headers['Content-Range'], "bytes 4096-8191/8192") + self.assertEqual(len(gotten_obj), 4096) + self.assertEqual(gotten_obj, self.aligned_obj[4096:8192]) + + def test_byte_0(self): + # Just the first byte, but it's index 0, so that's easy to get wrong + status, headers, gotten_obj = self._get_obj("bytes=0-0") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "1") + self.assertEqual(headers['Content-Range'], "bytes 0-0/14513") + self.assertEqual(gotten_obj, self.obj[0]) + + def test_unsatisfiable(self): + # Goes just one byte too far off the end of the object, so it's + # unsatisfiable + status, _junk, _junk = self._get_obj( + "bytes=%d-%d" % (len(self.obj), len(self.obj) + 100)) + self.assertEqual(status, 416) + + def test_off_end(self): + # Ranged GET that's mostly off the end of the object, but overlaps + # it in just the last byte + status, headers, gotten_obj = self._get_obj( + "bytes=%d-%d" % (len(self.obj) - 1, len(self.obj) + 100)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '1') + self.assertEqual(headers['Content-Range'], 'bytes 14512-14512/14513') + self.assertEqual(gotten_obj, self.obj[-1]) + + def test_aligned_off_end(self): + # Ranged GET that starts on a segment boundary but asks for a whole lot + status, headers, gotten_obj = self._get_obj( + "bytes=%d-%d" % (8192, len(self.obj) + 100)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '6321') + self.assertEqual(headers['Content-Range'], 'bytes 8192-14512/14513') + self.assertEqual(gotten_obj, self.obj[8192:]) + + def test_way_off_end(self): + # Ranged GET that's mostly off the end of the object, but overlaps + # it in just the last byte, and wants multiple segments' worth off + # the end + status, headers, gotten_obj = self._get_obj( + "bytes=%d-%d" % (len(self.obj) - 1, len(self.obj) * 1000)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '1') + self.assertEqual(headers['Content-Range'], 'bytes 14512-14512/14513') + self.assertEqual(gotten_obj, self.obj[-1]) + + def test_boundaries(self): + # Wants the last byte of segment 1 + the first byte of segment 2 + status, headers, gotten_obj = self._get_obj("bytes=4095-4096") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '2') + self.assertEqual(headers['Content-Range'], 'bytes 4095-4096/14513') + self.assertEqual(gotten_obj, self.obj[4095:4097]) + + def test_until_end(self): + # Wants the last byte of segment 1 + the rest + status, headers, gotten_obj = self._get_obj("bytes=4095-") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '10418') + self.assertEqual(headers['Content-Range'], 'bytes 4095-14512/14513') + self.assertEqual(gotten_obj, self.obj[4095:]) + + def test_small_suffix(self): + # Small range-suffix GET: the last 100 bytes (less than one segment) + status, headers, gotten_obj = self._get_obj("bytes=-100") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '100') + self.assertEqual(headers['Content-Range'], 'bytes 14413-14512/14513') + self.assertEqual(len(gotten_obj), 100) + self.assertEqual(gotten_obj, self.obj[-100:]) + + def test_small_suffix_aligned(self): + # Small range-suffix GET: the last 100 bytes, last segment is + # full-size + status, headers, gotten_obj = self._get_obj("bytes=-100", + self.aligned_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '100') + self.assertEqual(headers['Content-Range'], 'bytes 8092-8191/8192') + self.assertEqual(len(gotten_obj), 100) + + def test_suffix_two_segs(self): + # Ask for enough data that we need the last two segments. The last + # segment is short, though, so this ensures we compensate for that. + # + # Note that the total range size is less than one full-size segment. + suffix_len = len(self.obj) % self.seg_size + 1 + + status, headers, gotten_obj = self._get_obj("bytes=-%d" % suffix_len) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], str(suffix_len)) + self.assertEqual(headers['Content-Range'], + 'bytes %d-%d/%d' % (len(self.obj) - suffix_len, + len(self.obj) - 1, + len(self.obj))) + self.assertEqual(len(gotten_obj), suffix_len) + + def test_large_suffix(self): + # Large range-suffix GET: the last 5000 bytes (more than one segment) + status, headers, gotten_obj = self._get_obj("bytes=-5000") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '5000') + self.assertEqual(headers['Content-Range'], 'bytes 9513-14512/14513') + self.assertEqual(len(gotten_obj), 5000) + self.assertEqual(gotten_obj, self.obj[-5000:]) + + def test_overlarge_suffix(self): + # The last N+1 bytes of an N-byte object + status, headers, gotten_obj = self._get_obj( + "bytes=-%d" % (len(self.obj) + 1)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '14513') + self.assertEqual(headers['Content-Range'], 'bytes 0-14512/14513') + self.assertEqual(len(gotten_obj), len(self.obj)) + self.assertEqual(gotten_obj, self.obj) + + def test_small_suffix_tiny_object(self): + status, headers, gotten_obj = self._get_obj( + "bytes=-5", self.tiny_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '5') + self.assertEqual(headers['Content-Range'], 'bytes 12-16/17') + self.assertEqual(gotten_obj, self.tiny_obj[12:]) + + def test_overlarge_suffix_tiny_object(self): + status, headers, gotten_obj = self._get_obj( + "bytes=-1234567890", self.tiny_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '17') + self.assertEqual(headers['Content-Range'], 'bytes 0-16/17') + self.assertEqual(len(gotten_obj), len(self.tiny_obj)) + self.assertEqual(gotten_obj, self.tiny_obj) + + @patch_policies([ StoragePolicy(0, 'zero', True, object_ring=FakeRing(base_port=3000)), StoragePolicy(1, 'one', False, object_ring=FakeRing(base_port=3000)), @@ -5600,7 +6733,7 @@ class TestContainerController(unittest.TestCase): headers) self.assertEqual(int(headers ['X-Backend-Storage-Policy-Index']), - policy.idx) + int(policy)) # make sure all mocked responses are consumed self.assertRaises(StopIteration, mock_conn.code_iter.next) diff --git a/test/unit/proxy/test_sysmeta.py b/test/unit/proxy/test_sysmeta.py index d80f2855e..a45c689ab 100644 --- a/test/unit/proxy/test_sysmeta.py +++ b/test/unit/proxy/test_sysmeta.py @@ -135,7 +135,7 @@ class TestObjectSysmeta(unittest.TestCase): self.tmpdir = mkdtemp() self.testdir = os.path.join(self.tmpdir, 'tmp_test_object_server_ObjectController') - mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) + mkdirs(os.path.join(self.testdir, 'sda', 'tmp')) conf = {'devices': self.testdir, 'mount_check': 'false'} self.obj_ctlr = object_server.ObjectController( conf, logger=debug_logger('obj-ut')) |
