diff options
author | Guido van Rossum <guido@python.org> | 2014-01-10 17:04:32 -0800 |
---|---|---|
committer | Guido van Rossum <guido@python.org> | 2014-01-10 17:04:32 -0800 |
commit | 3cca58fb9fa5600e0c86ff94ce4eb48934456e6d (patch) | |
tree | 446f2ad44c78996e9abc31db5ae9177f0f72ce5b /examples | |
parent | 033da30e7ede771836a1687e509acf1195991083 (diff) | |
download | trollius-3cca58fb9fa5600e0c86ff94ce4eb48934456e6d.tar.gz |
Prune stale collections before closing oldest ones. Improve cloose logging.
Diffstat (limited to 'examples')
-rw-r--r-- | examples/crawl.py | 42 |
1 files changed, 36 insertions, 6 deletions
diff --git a/examples/crawl.py b/examples/crawl.py index 3a97966..f0c8e3d 100644 --- a/examples/crawl.py +++ b/examples/crawl.py @@ -159,8 +159,7 @@ class ConnectionPool: if not conns: del self.connections[key] if conn.stale(): - self.log(1, '(cached connection closed for %s)' % - repr(key)) + self.log(1, 'closing stale connection for', key) conn.close() # Just in case. else: self.log(1, '* Reusing pooled connection', key, @@ -181,26 +180,55 @@ class ConnectionPool: if conn.stale(): conn.close() return - conns = self.connections.setdefault(conn.key, []) + + key = conn.key + conns = self.connections.setdefault(key, []) conns.append(conn) self.queue.append(conn) - # TODO: Remove closed connections first. + if len(conns) <= self.max_tasks and len(self.queue) <= self.max_pool: + return + + # Prune the queue. + + # Close stale connections for this key first. + stale = [conn for conn in conns if conn.stale()] + if stale: + for conn in stale: + conns.remove(conn) + self.queue.remove(conn) + self.log(1, 'closing stale connection for', key) + conn.close() + if not conns: + del self.connections[key] # Close oldest connection(s) for this key if limit reached. while len(conns) > self.max_tasks: conn = conns.pop(0) - self.log(1, 'closing oldest connection for', conn.key) self.queue.remove(conn) + self.log(1, 'closing oldest connection for', key) conn.close() + if len(self.queue) <= self.max_pool: + return + + # Close overall stale connections. + stale = [conn for conn in self.queue if conn.stale()] + if stale: + for conn in stale: + conns = self.connections.get(conn.key) + conns.remove(conn) + self.queue.remove(conn) + self.log(1, 'closing stale connection for', key) + conn.close() + # Close oldest overall connection(s) if limit reached. while len(self.queue) > self.max_pool: conn = self.queue.pop(0) - self.log(1, 'closing oldest connection', conn.key) conns = self.connections.get(conn.key) c = conns.pop(0) assert conn == c, (conn.key, conn, c, conns) + self.log(1, 'closing overall oldest connection for', conn.key) conn.close() @@ -290,6 +318,8 @@ class Request: def close(self, recycle=False): """Close the connection, recycle if requested.""" if self.conn is not None: + if not recycle: + self.log(1, 'closing connection for', self.conn.key) self.conn.close(recycle) self.conn = None |