Process mirror retries before other queued requests.

Sometimes the ordering of downloads is important- eg Yum downloads
DRPMs before RPMs, so the delta rebuild can run while downloading..
Instead of adding retries at the end, process them in-place.

1 files changed, 13 insertions, 9 deletions

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index d06cdae..f3fd02f 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2170,6 +2170,7 @@ def parallel_wait(meter=None):
     dl = _ExternalDownloaderPool()
     host_con = {} # current host connection counts
     single = set() # hosts in single connection mode
+    retry_queue = []
 
     def start(opts, tries):
         opts.tries = tries
@@ -2250,7 +2251,7 @@ def parallel_wait(meter=None):
                     # mask this mirror and retry
                     if action.get('remove', 1):
                         removed.add(key)
-                    _async_queue.append(opts)
+                    retry_queue.append(opts)
                     continue
                 # fail=1 from callback
                 ug_err.errors = errors
@@ -2260,19 +2261,22 @@ def parallel_wait(meter=None):
             _run_callback(opts.failfunc, opts)
 
     try:
-        idx = 0
+        retry_idx = idx = 0
         while True:
-            if idx >= len(_async_queue):
-                # the queue is empty
+            if retry_idx < len(retry_queue):
+                # retries first
+                opts = retry_queue[retry_idx]
+                retry_idx += 1
+            elif idx < len(_async_queue):
+                # handle next request
+                opts = _async_queue[idx]
+                idx += 1
+            else:
+                # both queues are empty
                 if not dl.running: break
-                # pending dl may extend it
                 perform()
                 continue
 
-            # handle next request
-            opts = _async_queue[idx]
-            idx += 1
-
             # check global limit
             while len(dl.running) >= default_grabber.opts.max_connections:
                 perform()