From f86ef3bbdb5bffa1348a802e62b281d3f31d33ad Mon Sep 17 00:00:00 2001 From: Max Wittig Date: Mon, 8 Jun 2020 14:10:43 +0200 Subject: fix: use keyset pagination by default for /projects > 50000 Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/218504. Remove this in 13.1 --- gitlab/__init__.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/gitlab/__init__.py b/gitlab/__init__.py index f46cbac..705366a 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -16,13 +16,12 @@ # along with this program. If not, see . """Wrapper for the GitLab API.""" -from __future__ import print_function -from __future__ import absolute_import import importlib import time import warnings import requests +import requests.utils import gitlab.config from gitlab.const import * # noqa @@ -43,6 +42,8 @@ REDIRECT_MSG = ( "must update your GitLab URL to use https:// to avoid issues." ) +ALLOWED_KEYSET_ENDPOINTS = ["/projects"] + def _sanitize(value): if isinstance(value, dict): @@ -618,7 +619,7 @@ class Gitlab(object): Args: path (str): Path or full URL to query ('/projects' or - 'http://whatever/v4/api/projecs') + 'http://whatever/v4/api/projects') query_data (dict): Data to send as query parameters **kwargs: Extra options to send to the server (e.g. sudo, page, per_page) @@ -642,10 +643,22 @@ class Gitlab(object): get_all = kwargs.pop("all", False) url = self._build_url(path) + order_by = kwargs.get("order_by") + pagination = kwargs.get("pagination") + page = kwargs.get("page") + if ( + path in ALLOWED_KEYSET_ENDPOINTS + and (not order_by or order_by == "id") + and (not pagination or pagination == "keyset") + and not page + ): + kwargs["pagination"] = "keyset" + kwargs["order_by"] = "id" + if get_all is True and as_list is True: return list(GitlabList(self, url, query_data, **kwargs)) - if "page" in kwargs or as_list is True: + if page or as_list is True: # pagination requested, we return a list return list(GitlabList(self, url, query_data, get_next=False, **kwargs)) @@ -781,7 +794,14 @@ class GitlabList(object): query_data = query_data or {} result = self._gl.http_request("get", url, query_data=query_data, **kwargs) try: - self._next_url = result.links["next"]["url"] + links = result.links + if links: + next_url = links["next"]["url"] + else: + next_url = requests.utils.parse_header_links(result.headers["links"])[ + 0 + ]["url"] + self._next_url = next_url except KeyError: self._next_url = None self._current_page = result.headers.get("X-Page") -- cgit v1.2.1