From 5bc25292b8bb5b20ba2fff481fdc77b9909d0831 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Fri, 16 Aug 2019 11:04:10 -0700 Subject: Reduce internal client poll timeout for consumer iterator interface (#1824) More attempts to address heartbeat timing issues in consumers, especially with the iterator interface. Here we can reduce the `client.poll` timeout to at most the retry backoff (typically 100ms) so that the consumer iterator interface doesn't block for longer than the heartbeat timeout. --- kafka/consumer/group.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kafka/consumer/group.py b/kafka/consumer/group.py index 27ed4da..f9d0fb9 100644 --- a/kafka/consumer/group.py +++ b/kafka/consumer/group.py @@ -1086,9 +1086,7 @@ class KafkaConsumer(six.Iterator): partitions = self._subscription.missing_fetch_positions() self._update_fetch_positions(partitions) - poll_ms = 1000 * (self._consumer_timeout - time.time()) - if not self._fetcher.in_flight_fetches(): - poll_ms = min(poll_ms, self.config['reconnect_backoff_ms']) + poll_ms = min((1000 * (self._consumer_timeout - time.time())), self.config['retry_backoff_ms']) self._client.poll(timeout_ms=poll_ms) # after the long poll, we should check whether the group needs to rebalance -- cgit v1.2.1