From b0be02e1471c99e5e5e4bd52db1019006d26c349 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 25 May 2016 16:48:31 +0200 Subject: fix(remote): remove assertion in favour of runtime stability Fixes #442 --- git/remote.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/git/remote.py b/git/remote.py index e430abf5..54773a6f 100644 --- a/git/remote.py +++ b/git/remote.py @@ -20,8 +20,6 @@ from .refs import ( SymbolicReference, TagReference ) - - from git.util import ( LazyMixin, Iterable, @@ -35,6 +33,9 @@ from git.util import ( from git.cmd import handle_process_output from gitdb.util import join from git.compat import defenc +import logging + +log = logging.getLogger('git.remote') __all__ = ('RemoteProgress', 'PushInfo', 'FetchInfo', 'Remote') @@ -570,10 +571,16 @@ class Remote(LazyMixin, Iterable): fetch_head_info = [l.decode(defenc) for l in fp.readlines()] fp.close() - # NOTE: We assume to fetch at least enough progress lines to allow matching each fetch head line with it. l_fil = len(fetch_info_lines) l_fhi = len(fetch_head_info) - assert l_fil >= l_fhi, "len(%s) <= len(%s)" % (l_fil, l_fhi) + if l_fil >= l_fhi: + msg = "Fetch head does not contain enough lines to match with progress information\n" + msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" + msg += "Will ignore extra progress lines." + msg %= (l_fil, l_fhi) + log.warn(msg) + fetch_info_lines = fetch_info_lines[:l_fhi] + # end sanity check + sanitization output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) -- cgit v1.2.1 From 1537aabfa3bb32199e321766793c87864f36ee9a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 25 May 2016 18:11:32 +0200 Subject: fix(remote): better array truncation logic Previously, the logic was not correct. Now it should work either way, truncating the correct list to assure both always have the same length. Related to #442 --- git/remote.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/git/remote.py b/git/remote.py index 54773a6f..6a22768d 100644 --- a/git/remote.py +++ b/git/remote.py @@ -573,15 +573,19 @@ class Remote(LazyMixin, Iterable): l_fil = len(fetch_info_lines) l_fhi = len(fetch_head_info) - if l_fil >= l_fhi: - msg = "Fetch head does not contain enough lines to match with progress information\n" + if l_fil != l_fhi: + msg = "Fetch head lines do not match lines provided via progress information\n" msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" - msg += "Will ignore extra progress lines." + msg += "Will ignore extra progress lines or fetch head lines." msg %= (l_fil, l_fhi) log.warn(msg) - fetch_info_lines = fetch_info_lines[:l_fhi] + if l_fil < l_fhi: + fetch_head_info = fetch_head_info[:l_fil] + else: + fetch_info_lines = fetch_info_lines[:l_fhi] + # end truncate correct list # end sanity check + sanitization - + output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) return output -- cgit v1.2.1 From 9989f8965f34af5009361ec58f80bbf3ca75b465 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 May 2016 08:52:30 +0200 Subject: import OrderedDict from git.odict rather than directly from collections, to pix Py2.6 compatibility --- git/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index eef52534..bbbed62f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -13,7 +13,7 @@ import threading import errno import mmap -from collections import OrderedDict +from git.odict import OrderedDict from contextlib import contextmanager import signal -- cgit v1.2.1 From b40d4b54e09a546dd9514b63c0cb141c64d80384 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 26 May 2016 09:18:51 +0200 Subject: chore(compat): re-add allowed breakage of py2.6 As inspired by comments in #431 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5c01b3fd..17c34104 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,14 @@ language: python python: + - "2.6" - "2.7" - "3.3" - "3.4" - "3.5" # - "pypy" - won't work as smmap doesn't work (see gitdb/.travis.yml for details) +matrix: + allow_failures: + python: "2.6" git: # a higher depth is needed for most of the tests - must be high enough to not actually be shallow # as we clone our own repository in the process -- cgit v1.2.1 From b9a7dc5fe98e1aa666445bc240055b21ed809824 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 26 May 2016 09:24:53 +0200 Subject: chore(compat): another attempt to get travis right --- .travis.yml | 2 +- doc/source/intro.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17c34104..99ecd4aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ python: # - "pypy" - won't work as smmap doesn't work (see gitdb/.travis.yml for details) matrix: allow_failures: - python: "2.6" + - python: "2.6" git: # a higher depth is needed for most of the tests - must be high enough to not actually be shallow # as we clone our own repository in the process diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 647323c4..6c4e50f5 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -14,7 +14,8 @@ Requirements ============ * `Python`_ 2.7 or newer - Since GitPython 2.0.0 + Since GitPython 2.0.0. Please note that python 2.6 is still reasonably well supported, but might + deteriorate over time. * `Git`_ 1.7.0 or newer It should also work with older versions, but it may be that some operations involving remotes will not work as expected. -- cgit v1.2.1 From 04ff96ddd0215881f72cc532adc6ff044e77ea3e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 26 May 2016 18:52:38 +0200 Subject: fix(remote): real-time reading of lines from stderr That way, progress usage will behave as expected. Fixes #444 --- git/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index 6a22768d..bff26459 100644 --- a/git/remote.py +++ b/git/remote.py @@ -550,7 +550,7 @@ class Remote(LazyMixin, Iterable): progress_handler = progress.new_message_handler() - for line in proc.stderr.readlines(): + for line in proc.stderr: line = line.decode(defenc) for pline in progress_handler(line): if line.startswith('fatal:') or line.startswith('error:'): -- cgit v1.2.1 From 515a6b9ccf87bd1d3f5f2edd229d442706705df5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 26 May 2016 19:41:00 +0200 Subject: fix(remote): use universal_newlines for fetch/push That way, real-time parsing of output should finally be possible. Related to #444 --- git/cmd.py | 9 +++++++-- git/remote.py | 10 ++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index bbbed62f..0c3cc8ca 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -44,7 +44,8 @@ from git.compat import ( execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output', 'with_exceptions', 'as_process', 'stdout_as_string', - 'output_stream', 'with_stdout', 'kill_after_timeout') + 'output_stream', 'with_stdout', 'kill_after_timeout', + 'universal_newlines') log = logging.getLogger('git.cmd') log.addHandler(logging.NullHandler()) @@ -487,6 +488,7 @@ class Git(LazyMixin): stdout_as_string=True, kill_after_timeout=None, with_stdout=True, + universal_newlines=False, **subprocess_kwargs ): """Handles executing the command on the shell and consumes and returns @@ -541,7 +543,9 @@ class Git(LazyMixin): specify may not be the same ones. :param with_stdout: If True, default True, we open stdout on the created process - + :param universal_newlines: + if True, pipes will be opened as text, and lines are split at + all known line endings. :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process should be killed. This will have no effect if as_process is set to True. It is @@ -608,6 +612,7 @@ class Git(LazyMixin): stdout=with_stdout and PIPE or open(os.devnull, 'wb'), shell=self.USE_SHELL, close_fds=(os.name == 'posix'), # unsupported on windows + universal_newlines=universal_newlines, **subprocess_kwargs ) except cmd_not_found_exception as err: diff --git a/git/remote.py b/git/remote.py index bff26459..169d4f79 100644 --- a/git/remote.py +++ b/git/remote.py @@ -663,8 +663,8 @@ class Remote(LazyMixin, Iterable): else: args = [refspec] - proc = self.repo.git.fetch(self, *args, as_process=True, with_stdout=False, v=True, - **kwargs) + proc = self.repo.git.fetch(self, *args, as_process=True, with_stdout=False, + universal_newlines=True, v=True, **kwargs) res = self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) if hasattr(self.repo.odb, 'update_cache'): self.repo.odb.update_cache() @@ -682,7 +682,8 @@ class Remote(LazyMixin, Iterable): # No argument refspec, then ensure the repo's config has a fetch refspec. self._assert_refspec() kwargs = add_progress(kwargs, self.repo.git, progress) - proc = self.repo.git.pull(self, refspec, with_stdout=False, as_process=True, v=True, **kwargs) + proc = self.repo.git.pull(self, refspec, with_stdout=False, as_process=True, + universal_newlines=True, v=True, **kwargs) res = self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) if hasattr(self.repo.odb, 'update_cache'): self.repo.odb.update_cache() @@ -707,7 +708,8 @@ class Remote(LazyMixin, Iterable): If the operation fails completely, the length of the returned IterableList will be null.""" kwargs = add_progress(kwargs, self.repo.git, progress) - proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, **kwargs) + proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, + universal_newlines=True, **kwargs) return self._get_push_info(proc, progress or RemoteProgress()) @property -- cgit v1.2.1 From 5efdad2502098a2bd3af181931dc011501a13904 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 26 May 2016 19:50:05 +0200 Subject: fix(remote): py3 compatibility --- git/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/remote.py b/git/remote.py index 169d4f79..b440e2cc 100644 --- a/git/remote.py +++ b/git/remote.py @@ -32,7 +32,7 @@ from git.util import ( ) from git.cmd import handle_process_output from gitdb.util import join -from git.compat import defenc +from git.compat import (defenc, safe_decode) import logging log = logging.getLogger('git.remote') @@ -551,7 +551,7 @@ class Remote(LazyMixin, Iterable): progress_handler = progress.new_message_handler() for line in proc.stderr: - line = line.decode(defenc) + line = safe_decode(line) for pline in progress_handler(line): if line.startswith('fatal:') or line.startswith('error:'): raise GitCommandError(("Error when fetching: %s" % line,), 2) -- cgit v1.2.1 From 902679c47c3d1238833ac9c9fdbc7c0ddbedf509 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 26 May 2016 19:51:09 +0200 Subject: fix(remote): py3 compatibility Related to #444 --- git/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/remote.py b/git/remote.py index b440e2cc..92203588 100644 --- a/git/remote.py +++ b/git/remote.py @@ -32,7 +32,7 @@ from git.util import ( ) from git.cmd import handle_process_output from gitdb.util import join -from git.compat import (defenc, safe_decode) +from git.compat import (defenc, force_text) import logging log = logging.getLogger('git.remote') @@ -551,7 +551,7 @@ class Remote(LazyMixin, Iterable): progress_handler = progress.new_message_handler() for line in proc.stderr: - line = safe_decode(line) + line = force_text(line) for pline in progress_handler(line): if line.startswith('fatal:') or line.startswith('error:'): raise GitCommandError(("Error when fetching: %s" % line,), 2) -- cgit v1.2.1 From 33940022821ec5e1c1766eb60ffd80013cb12771 Mon Sep 17 00:00:00 2001 From: Guyzmo Date: Thu, 26 May 2016 20:34:01 +0200 Subject: Changing warning to debug logging, to avoid warning showing off when nothing's wrong cf #444 Signed-off-by: Guyzmo --- git/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index 92203588..88658c38 100644 --- a/git/remote.py +++ b/git/remote.py @@ -578,7 +578,7 @@ class Remote(LazyMixin, Iterable): msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" msg += "Will ignore extra progress lines or fetch head lines." msg %= (l_fil, l_fhi) - log.warn(msg) + log.debug(msg) if l_fil < l_fhi: fetch_head_info = fetch_head_info[:l_fil] else: -- cgit v1.2.1 From 39164b038409cb66960524e19f60e83d68790325 Mon Sep 17 00:00:00 2001 From: Aleksander Nitecki Date: Thu, 26 May 2016 23:51:19 +0200 Subject: Use proper syntax for conditional expression (instead of abusing the "short-circuit" property of logical operations) --- git/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index 0c3cc8ca..c29e3485 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -609,7 +609,7 @@ class Git(LazyMixin): bufsize=-1, stdin=istream, stderr=PIPE, - stdout=with_stdout and PIPE or open(os.devnull, 'wb'), + stdout=PIPE if with_stdout else open(os.devnull, 'wb'), shell=self.USE_SHELL, close_fds=(os.name == 'posix'), # unsupported on windows universal_newlines=universal_newlines, -- cgit v1.2.1