First maybe-working version of process pools.

1 files changed, 4 insertions, 4 deletions

diff --git a/crawl.py b/crawl.py
index 57f9e8e..74c4436 100644
--- a/crawl.py
+++ b/crawl.py
@@ -11,7 +11,7 @@ URLS = ['http://www.google.com/',
         'http://www.thisurlprobablydoesnotexist.com',
         'http://www.slashdot.org/',
         'http://www.python.org/',
-        'http://www.sweetapp.com/']
+        'http://www.sweetapp.com/'] * 1000
 
 def load_url(url, timeout):
     return urllib.request.urlopen(url, timeout=timeout).read()
@@ -25,14 +25,14 @@ def download_urls(urls, timeout=60):
             pass
     return url_to_content
 
-executor = futures.thread.ThreadPoolExecutor(max_threads=100)
+executor = futures.ProcessPoolExecutor(100)
 def download_urls_with_futures(urls, timeout=60):
     url_to_content = {}
     fs = executor.run(
             (functools.partial(load_url, url, timeout) for url in urls),
             timeout=timeout)
-    for url, future in zip(urls, fs.result_futures()):
+    for url, future in zip(urls, fs.successful_futures()):
         url_to_content[url] = future.result()
     return url_to_content
 
-print(download_urls(URLS))
+print(download_urls_with_futures(URLS))