From 331ef6f6116531c0a8a6ddb737a44fa6e3c886ed Mon Sep 17 00:00:00 2001 From: Zdenek Pavlas Date: Mon, 9 Dec 2013 16:09:23 +0100 Subject: Process mirror retries before other queued requests. Sometimes the ordering of downloads is important- eg Yum downloads DRPMs before RPMs, so the delta rebuild can run while downloading.. Instead of adding retries at the end, process them in-place. --- urlgrabber/grabber.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'urlgrabber') diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py index d06cdae..f3fd02f 100644 --- a/urlgrabber/grabber.py +++ b/urlgrabber/grabber.py @@ -2170,6 +2170,7 @@ def parallel_wait(meter=None): dl = _ExternalDownloaderPool() host_con = {} # current host connection counts single = set() # hosts in single connection mode + retry_queue = [] def start(opts, tries): opts.tries = tries @@ -2250,7 +2251,7 @@ def parallel_wait(meter=None): # mask this mirror and retry if action.get('remove', 1): removed.add(key) - _async_queue.append(opts) + retry_queue.append(opts) continue # fail=1 from callback ug_err.errors = errors @@ -2260,19 +2261,22 @@ def parallel_wait(meter=None): _run_callback(opts.failfunc, opts) try: - idx = 0 + retry_idx = idx = 0 while True: - if idx >= len(_async_queue): - # the queue is empty + if retry_idx < len(retry_queue): + # retries first + opts = retry_queue[retry_idx] + retry_idx += 1 + elif idx < len(_async_queue): + # handle next request + opts = _async_queue[idx] + idx += 1 + else: + # both queues are empty if not dl.running: break - # pending dl may extend it perform() continue - # handle next request - opts = _async_queue[idx] - idx += 1 - # check global limit while len(dl.running) >= default_grabber.opts.max_connections: perform() -- cgit v1.2.1