Shuffle download order (of containers and objects)

When downloading the same containers or objects with multiple
invocations of the swift command-line client, you'll get better
throughput and avoid "hot spots" if each client randomizes its download
order.

Note that the marker must be picked *before* shuffling the containers or
objects.

Change-Id: I7240eda57a80e1708c2483827c6329fd57d5fc51

1 files changed, 5 insertions, 2 deletions

diff --git a/bin/swift b/bin/swift
index 11f9553..b198594 100755
--- a/bin/swift
+++ b/bin/swift
@@ -22,6 +22,7 @@ from optparse import OptionParser
 from os import environ, listdir, makedirs, utime, _exit as os_exit
 from os.path import basename, dirname, getmtime, getsize, isdir, join
 from Queue import Empty, Queue
+from random import shuffle
 from sys import argv, exc_info, exit, stderr, stdout
 from threading import current_thread, enumerate as threading_enumerate, Thread
 from time import sleep
@@ -423,9 +424,10 @@ def st_download(parser, args, print_queue, error_queue):
                            conn.get_container(container, marker=marker)[1]]
                 if not objects:
                     break
+                marker = objects[-1]
+                shuffle(objects)
                 for obj in objects:
                     object_queue.put((container, obj))
-                marker = objects[-1]
         except ClientException, err:
             if err.http_status != 404:
                 raise
@@ -450,9 +452,10 @@ def st_download(parser, args, print_queue, error_queue):
                               for c in conn.get_account(marker=marker)[1]]
                 if not containers:
                     break
+                marker = containers[-1]
+                shuffle(containers)
                 for container in containers:
                     container_queue.put(container)
-                marker = containers[-1]
         except ClientException, err:
             if err.http_status != 404:
                 raise