diff options
| -rw-r--r-- | .github/FUNDING.yml | 1 | ||||
| -rw-r--r-- | CONTRIBUTING.md | 3 | ||||
| -rw-r--r-- | MANIFEST.in | 7 | ||||
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | VERSION | 2 | ||||
| -rw-r--r-- | doc/source/changes.rst | 13 | ||||
| -rw-r--r-- | git/__init__.py | 9 | ||||
| -rw-r--r-- | git/cmd.py | 56 | ||||
| -rw-r--r-- | git/compat/__init__.py (renamed from git/compat.py) | 39 | ||||
| -rw-r--r-- | git/compat/typing.py | 13 | ||||
| -rw-r--r-- | git/config.py | 7 | ||||
| -rw-r--r-- | git/db.py | 17 | ||||
| -rw-r--r-- | git/diff.py | 126 | ||||
| -rw-r--r-- | git/exc.py | 17 | ||||
| -rw-r--r-- | git/objects/__init__.py | 4 | ||||
| -rw-r--r-- | git/objects/base.py | 3 | ||||
| -rw-r--r-- | git/refs/reference.py | 4 | ||||
| -rw-r--r-- | git/refs/symbolic.py | 2 | ||||
| -rw-r--r-- | git/remote.py | 37 | ||||
| -rw-r--r-- | git/repo/base.py | 151 | ||||
| -rw-r--r-- | git/repo/fun.py | 21 | ||||
| -rw-r--r-- | git/types.py | 18 | ||||
| -rw-r--r-- | git/util.py | 47 | ||||
| -rw-r--r-- | mypy.ini | 7 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | test-requirements.txt | 2 | ||||
| -rw-r--r-- | test/fixtures/diff_file_with_colon | bin | 0 -> 351 bytes | |||
| -rw-r--r-- | test/test_clone.py | 32 | ||||
| -rw-r--r-- | test/test_diff.py | 7 | ||||
| -rw-r--r-- | test/test_repo.py | 15 | ||||
| -rw-r--r-- | test/test_util.py | 20 | ||||
| -rw-r--r-- | tox.ini | 11 |
32 files changed, 453 insertions, 243 deletions
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..80819f5d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: byron diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4217cbaf..f685e7e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,6 +5,7 @@ The following is a short step-by-step rundown of what one typically would do to * [fork this project](https://github.com/gitpython-developers/GitPython/fork) on GitHub. * For setting up the environment to run the self tests, please look at `.travis.yml`. * Please try to **write a test that fails unless the contribution is present.** -* Feel free to add yourself to AUTHORS file. +* Try to avoid massive commits and prefer to take small steps, with one commit for each. +* Feel free to add yourself to AUTHORS file. * Create a pull request. diff --git a/MANIFEST.in b/MANIFEST.in index 5fd771db..f02721fc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,11 @@ -include VERSION -include LICENSE -include CHANGES include AUTHORS +include CHANGES include CONTRIBUTING.md +include LICENSE include README.md +include VERSION include requirements.txt +include test-requirements.txt recursive-include doc * recursive-exclude test * @@ -15,7 +15,7 @@ release: clean make force_release force_release: clean - git push --tags origin master + git push --tags origin main python3 setup.py sdist bdist_wheel twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* @@ -24,7 +24,7 @@ docker-build: test: docker-build # NOTE!!! - # NOTE!!! If you are not running from master or have local changes then tests will fail + # NOTE!!! If you are not running from main or have local changes then tests will fail # NOTE!!! docker run --rm -v ${CURDIR}:/src -w /src -t gitpython:xenial tox @@ -1 +1 @@ -3.1.13 +3.1.15 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 405179d0..1b916f30 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,7 +2,15 @@ Changelog ========= -3.1.?? +3.1.15 +====== + +* add deprectation warning for python 3.5 + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/47?closed=1 + +3.1.14 ====== * git.Commit objects now have a ``replace`` method that will return a @@ -10,6 +18,9 @@ Changelog * Add python 3.9 support * Drop python 3.4 support +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/46?closed=1 + 3.1.13 ====== diff --git a/git/__init__.py b/git/__init__.py index e2f960db..ae9254a2 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -5,6 +5,7 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php # flake8: noqa #@PydevCodeAnalysisIgnore +from git.exc import * # @NoMove @IgnorePep8 import inspect import os import sys @@ -16,8 +17,6 @@ from git.types import PathLike __version__ = 'git' - - #{ Initialization def _init_externals() -> None: """Initialize external projects by putting them into the path""" @@ -32,13 +31,13 @@ def _init_externals() -> None: #} END initialization + ################# _init_externals() ################# #{ Imports -from git.exc import * # @NoMove @IgnorePep8 try: from git.config import GitConfigParser # @NoMove @IgnorePep8 from git.objects import * # @NoMove @IgnorePep8 @@ -68,7 +67,8 @@ __all__ = [name for name, obj in locals().items() #{ Initialize git executable path GIT_OK = None -def refresh(path:Optional[PathLike]=None) -> None: + +def refresh(path: Optional[PathLike] = None) -> None: """Convenience method for setting the git executable path.""" global GIT_OK GIT_OK = False @@ -81,6 +81,7 @@ def refresh(path:Optional[PathLike]=None) -> None: GIT_OK = True #} END initialize git executable path + ################# try: refresh() @@ -19,7 +19,7 @@ import sys import threading from collections import OrderedDict from textwrap import dedent -from typing import Any, Dict, List, Optional +import warnings from git.compat import ( defenc, @@ -29,7 +29,7 @@ from git.compat import ( is_win, ) from git.exc import CommandError -from git.util import is_cygwin_git, cygpath, expand_path +from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present from .exc import ( GitCommandError, @@ -40,8 +40,6 @@ from .util import ( stream_copy, ) -from .types import PathLike - execute_kwargs = {'istream', 'with_extended_output', 'with_exceptions', 'as_process', 'stdout_as_string', 'output_stream', 'with_stdout', 'kill_after_timeout', @@ -85,8 +83,8 @@ def handle_process_output(process, stdout_handler, stderr_handler, line = line.decode(defenc) handler(line) except Exception as ex: - log.error("Pumping %r of cmd(%s) failed due to: %r", name, cmdline, ex) - raise CommandError(['<%s-pump>' % name] + cmdline, ex) from ex + log.error("Pumping %r of cmd(%s) failed due to: %r", name, remove_password_if_present(cmdline), ex) + raise CommandError(['<%s-pump>' % name] + remove_password_if_present(cmdline), ex) from ex finally: stream.close() @@ -105,7 +103,7 @@ def handle_process_output(process, stdout_handler, stderr_handler, for name, stream, handler in pumps: t = threading.Thread(target=pump_stream, args=(cmdline, name, stream, decode_streams, handler)) - t.setDaemon(True) + t.daemon = True t.start() threads.append(t) @@ -140,7 +138,7 @@ CREATE_NO_WINDOW = 0x08000000 ## CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards, # see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal -PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP +PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined] if is_win else 0) @@ -212,7 +210,7 @@ class Git(LazyMixin): # - a GitCommandNotFound error is spawned by ourselves # - a PermissionError is spawned if the git executable provided # cannot be executed for whatever reason - + has_git = False try: cls().version() @@ -408,7 +406,7 @@ class Git(LazyMixin): if status != 0: errstr = read_all_from_possibly_closed_stream(self.proc.stderr) log.debug('AutoInterrupt wait stderr: %r' % (errstr,)) - raise GitCommandError(self.args, status, errstr) + raise GitCommandError(remove_password_if_present(self.args), status, errstr) # END status handling return status # END auto interrupt @@ -500,7 +498,7 @@ class Git(LazyMixin): # skipcq: PYL-E0301 def __iter__(self): return self - + def __next__(self): return self.next() @@ -519,7 +517,7 @@ class Git(LazyMixin): self._stream.read(bytes_left + 1) # END handle incomplete read - def __init__(self, working_dir: Optional[PathLike]=None) -> None: + def __init__(self, working_dir=None): """Initialize this instance with: :param working_dir: @@ -528,12 +526,12 @@ class Git(LazyMixin): It is meant to be the working tree directory if available, or the .git directory in case of bare repositories.""" super(Git, self).__init__() - self._working_dir = expand_path(working_dir) if working_dir is not None else None + self._working_dir = expand_path(working_dir) self._git_options = () - self._persistent_git_options = [] # type: List[str] + self._persistent_git_options = [] # Extra environment variables to pass to git commands - self._environment = {} # type: Dict[str, Any] + self._environment = {} # cached command slots self.cat_file_header = None @@ -547,7 +545,7 @@ class Git(LazyMixin): return LazyMixin.__getattr__(self, name) return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) - def set_persistent_git_options(self, **kwargs) -> None: + def set_persistent_git_options(self, **kwargs): """Specify command line options to the git executable for subsequent subcommand calls @@ -641,7 +639,7 @@ class Git(LazyMixin): :param env: A dictionary of environment variables to be passed to `subprocess.Popen`. - + :param max_chunk_size: Maximum number of bytes in one chunk of data passed to the output_stream in one invocation of write() method. If the given number is not positive then @@ -685,8 +683,10 @@ class Git(LazyMixin): :note: If you add additional keyword arguments to the signature of this method, you must update the execute_kwargs tuple housed in this module.""" + # Remove password for the command if present + redacted_command = remove_password_if_present(command) if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != 'full' or as_process): - log.info(' '.join(command)) + log.info(' '.join(redacted_command)) # Allow the user to have the command executed in their working dir. cwd = self._working_dir or os.getcwd() @@ -707,7 +707,7 @@ class Git(LazyMixin): if is_win: cmd_not_found_exception = OSError if kill_after_timeout: - raise GitCommandError(command, '"kill_after_timeout" feature is not supported on Windows.') + raise GitCommandError(redacted_command, '"kill_after_timeout" feature is not supported on Windows.') else: if sys.version_info[0] > 2: cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable @@ -722,7 +722,7 @@ class Git(LazyMixin): if istream: istream_ok = "<valid stream>" log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", - command, cwd, universal_newlines, shell, istream_ok) + redacted_command, cwd, universal_newlines, shell, istream_ok) try: proc = Popen(command, env=env, @@ -738,7 +738,7 @@ class Git(LazyMixin): **subprocess_kwargs ) except cmd_not_found_exception as err: - raise GitCommandNotFound(command, err) from err + raise GitCommandNotFound(redacted_command, err) from err if as_process: return self.AutoInterrupt(proc, command) @@ -788,7 +788,7 @@ class Git(LazyMixin): watchdog.cancel() if kill_check.isSet(): stderr_value = ('Timeout: the command "%s" did not complete in %d ' - 'secs.' % (" ".join(command), kill_after_timeout)) + 'secs.' % (" ".join(redacted_command), kill_after_timeout)) if not universal_newlines: stderr_value = stderr_value.encode(defenc) # strip trailing "\n" @@ -812,7 +812,7 @@ class Git(LazyMixin): proc.stderr.close() if self.GIT_PYTHON_TRACE == 'full': - cmdstr = " ".join(command) + cmdstr = " ".join(redacted_command) def as_text(stdout_value): return not output_stream and safe_decode(stdout_value) or '<OUTPUT_STREAM>' @@ -828,7 +828,7 @@ class Git(LazyMixin): # END handle debug printing if with_exceptions and status != 0: - raise GitCommandError(command, status, stderr_value, stdout_value) + raise GitCommandError(redacted_command, status, stderr_value, stdout_value) if isinstance(stdout_value, bytes) and stdout_as_string: # could also be output_stream stdout_value = safe_decode(stdout_value) @@ -905,8 +905,14 @@ class Git(LazyMixin): def transform_kwargs(self, split_single_char_options=True, **kwargs): """Transforms Python style kwargs into git command line options.""" + # Python 3.6 preserves the order of kwargs and thus has a stable + # order. For older versions sort the kwargs by the key to get a stable + # order. + if sys.version_info[:2] < (3, 6): + kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0])) + warnings.warn("Python 3.5 support is deprecated and will be removed 2021-09-05.\n" + + "It does not preserve the order for key-word arguments and enforce lexical sorting instead.") args = [] - kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0])) for k, v in kwargs.items(): if isinstance(v, (list, tuple)): for value in v: diff --git a/git/compat.py b/git/compat/__init__.py index 4fe394ae..c4bd2aa3 100644 --- a/git/compat.py +++ b/git/compat/__init__.py @@ -16,9 +16,22 @@ from gitdb.utils.encoding import ( force_text # @UnusedImport ) -from typing import Any, AnyStr, Dict, Optional, Type +# typing -------------------------------------------------------------------- + +from typing import ( + Any, + AnyStr, + Dict, + IO, + Optional, + Type, + Union, + overload, +) from git.types import TBD +# --------------------------------------------------------------------------- + is_win = (os.name == 'nt') # type: bool is_posix = (os.name == 'posix') @@ -26,7 +39,13 @@ is_darwin = (os.name == 'darwin') defenc = sys.getfilesystemencoding() -def safe_decode(s: Optional[AnyStr]) -> Optional[str]: +@overload +def safe_decode(s: None) -> None: ... + +@overload +def safe_decode(s: Union[IO[str], AnyStr]) -> str: ... + +def safe_decode(s: Union[IO[str], AnyStr, None]) -> Optional[str]: """Safely decodes a binary string to unicode""" if isinstance(s, str): return s @@ -38,6 +57,11 @@ def safe_decode(s: Optional[AnyStr]) -> Optional[str]: raise TypeError('Expected bytes or text, but got %r' % (s,)) +@overload +def safe_encode(s: None) -> None: ... + +@overload +def safe_encode(s: AnyStr) -> bytes: ... def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: """Safely encodes a binary string to unicode""" @@ -51,6 +75,12 @@ def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: raise TypeError('Expected bytes or text, but got %r' % (s,)) +@overload +def win_encode(s: None) -> None: ... + +@overload +def win_encode(s: AnyStr) -> bytes: ... + def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: """Encode unicodes for process arguments on Windows.""" if isinstance(s, str): @@ -62,9 +92,9 @@ def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: return None -def with_metaclass(meta: Type[Any], *bases: Any) -> 'metaclass': # type: ignore ## mypy cannot understand dynamic class creation +def with_metaclass(meta: Type[Any], *bases: Any) -> 'metaclass': # type: ignore ## mypy cannot understand dynamic class creation """copied from https://github.com/Byron/bcore/blob/master/src/python/butility/future.py#L15""" - + class metaclass(meta): # type: ignore __call__ = type.__call__ __init__ = type.__init__ # type: ignore @@ -75,4 +105,3 @@ def with_metaclass(meta: Type[Any], *bases: Any) -> 'metaclass': # type: ignore return meta(name, bases, d) return metaclass(meta.__name__ + 'Helper', None, {}) - diff --git a/git/compat/typing.py b/git/compat/typing.py new file mode 100644 index 00000000..925c5ba2 --- /dev/null +++ b/git/compat/typing.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# config.py +# Copyright (C) 2021 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php + +import sys + +if sys.version_info[:2] >= (3, 8): + from typing import Final, Literal # noqa: F401 +else: + from typing_extensions import Final, Literal # noqa: F401 diff --git a/git/config.py b/git/config.py index ffbbfab4..0c8d975d 100644 --- a/git/config.py +++ b/git/config.py @@ -16,14 +16,13 @@ import re import fnmatch from collections import OrderedDict -from typing_extensions import Literal - from git.compat import ( defenc, force_text, with_metaclass, is_win, ) +from git.compat.typing import Literal from git.util import LockFile import os.path as osp @@ -196,7 +195,7 @@ class _OMD(OrderedDict): return [(k, self.getall(k)) for k in self] -def get_config_path(config_level: Literal['system','global','user','repository']) -> str: +def get_config_path(config_level: Literal['system', 'global', 'user', 'repository']) -> str: # we do not support an absolute path of the gitconfig on windows , # use the global config instead @@ -216,7 +215,7 @@ def get_config_path(config_level: Literal['system','global','user','repository'] raise ValueError("Invalid configuration level: %r" % config_level) -class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): +class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): # type: ignore ## mypy does not understand dynamic class creation # noqa: E501 """Implements specifics required to read git style configuration files. @@ -1,5 +1,4 @@ """Module with our own gitdb implementation - it uses the git command""" -from typing import AnyStr from git.util import bin_to_hex, hex_to_bin from gitdb.base import ( OInfo, @@ -7,21 +6,23 @@ from gitdb.base import ( ) from gitdb.db import GitDB # @UnusedImport from gitdb.db import LooseObjectDB -from gitdb.exc import BadObject -from .exc import GitCommandError +from gitdb.exc import BadObject +from git.exc import GitCommandError # typing------------------------------------------------- -from .cmd import Git -from .types import PathLike +from typing import TYPE_CHECKING, AnyStr +from git.types import PathLike + +if TYPE_CHECKING: + from git.cmd import Git + # -------------------------------------------------------- __all__ = ('GitCmdObjectDB', 'GitDB') -# class GitCmdObjectDB(CompoundDB, ObjectDBW): - class GitCmdObjectDB(LooseObjectDB): @@ -33,7 +34,7 @@ class GitCmdObjectDB(LooseObjectDB): have packs and the other implementations """ - def __init__(self, root_path: PathLike, git: Git) -> None: + def __init__(self, root_path: PathLike, git: 'Git') -> None: """Initialize this instance with the root and a git command""" super(GitCmdObjectDB, self).__init__(root_path) self._git = git diff --git a/git/diff.py b/git/diff.py index b25aadc7..943916ea 100644 --- a/git/diff.py +++ b/git/diff.py @@ -15,11 +15,14 @@ from .objects.util import mode_str_to_int # typing ------------------------------------------------------------------ -from .objects.tree import Tree -from git.repo.base import Repo -from typing_extensions import Final, Literal +from typing import Any, Iterator, List, Match, Optional, Tuple, Type, Union, TYPE_CHECKING +from git.compat.typing import Final, Literal from git.types import TBD -from typing import Any, Iterator, List, Match, Optional, Tuple, Type, Union + +if TYPE_CHECKING: + from .objects.tree import Tree + from git.repo.base import Repo + Lit_change_type = Literal['A', 'D', 'M', 'R', 'T'] # ------------------------------------------------------------------------ @@ -27,7 +30,7 @@ Lit_change_type = Literal['A', 'D', 'M', 'R', 'T'] __all__ = ('Diffable', 'DiffIndex', 'Diff', 'NULL_TREE') # Special object to compare against the empty tree in diffs -NULL_TREE: Final[object] = object() +NULL_TREE = object() # type: Final[object] _octal_byte_re = re.compile(b'\\\\([0-9]{3})') @@ -79,7 +82,7 @@ class Diffable(object): Subclasses can use it to alter the behaviour of the superclass""" return args - def diff(self, other: Union[Type[Index], Type[Tree], object, None, str] = Index, + def diff(self, other: Union[Type[Index], Type['Tree'], object, None, str] = Index, paths: Union[str, List[str], Tuple[str, ...], None] = None, create_patch: bool = False, **kwargs: Any) -> 'DiffIndex': """Creates diffs between two items being trees, trees and index or an @@ -271,7 +274,7 @@ class Diff(object): "new_file", "deleted_file", "copied_file", "raw_rename_from", "raw_rename_to", "diff", "change_type", "score") - def __init__(self, repo: Repo, + def __init__(self, repo: 'Repo', a_rawpath: Optional[bytes], b_rawpath: Optional[bytes], a_blob_id: Union[str, bytes, None], b_blob_id: Union[str, bytes, None], a_mode: Union[bytes, str, None], b_mode: Union[bytes, str, None], @@ -425,7 +428,7 @@ class Diff(object): return None @classmethod - def _index_from_patch_format(cls, repo: Repo, proc: TBD) -> DiffIndex: + def _index_from_patch_format(cls, repo: 'Repo', proc: TBD) -> DiffIndex: """Create a new DiffIndex from the given text which must be in patch format :param repo: is the repository we are operating on - it is required :param stream: result of 'git diff' as a stream (supporting file protocol) @@ -487,6 +490,58 @@ class Diff(object): return index + @staticmethod + def _handle_diff_line(lines_bytes: bytes, repo: 'Repo', index: TBD) -> None: + lines = lines_bytes.decode(defenc) + + for line in lines.split(':')[1:]: + meta, _, path = line.partition('\x00') + path = path.rstrip('\x00') + a_blob_id, b_blob_id = None, None # Type: Optional[str] + old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4) + # Change type can be R100 + # R: status letter + # 100: score (in case of copy and rename) + change_type = _change_type[0] + score_str = ''.join(_change_type[1:]) + score = int(score_str) if score_str.isdigit() else None + path = path.strip() + a_path = path.encode(defenc) + b_path = path.encode(defenc) + deleted_file = False + new_file = False + copied_file = False + rename_from = None + rename_to = None + + # NOTE: We cannot conclude from the existence of a blob to change type + # as diffs with the working do not have blobs yet + if change_type == 'D': + b_blob_id = None # Optional[str] + deleted_file = True + elif change_type == 'A': + a_blob_id = None + new_file = True + elif change_type == 'C': + copied_file = True + a_path_str, b_path_str = path.split('\x00', 1) + a_path = a_path_str.encode(defenc) + b_path = b_path_str.encode(defenc) + elif change_type == 'R': + a_path_str, b_path_str = path.split('\x00', 1) + a_path = a_path_str.encode(defenc) + b_path = b_path_str.encode(defenc) + rename_from, rename_to = a_path, b_path + elif change_type == 'T': + # Nothing to do + pass + # END add/remove handling + + diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, + new_file, deleted_file, copied_file, rename_from, rename_to, + '', change_type, score) + index.append(diff) + @classmethod def _index_from_raw_format(cls, repo: 'Repo', proc: TBD) -> DiffIndex: """Create a new DiffIndex from the given stream which must be in raw format. @@ -495,58 +550,7 @@ class Diff(object): # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() - - def handle_diff_line(lines_bytes: bytes) -> None: - lines = lines_bytes.decode(defenc) - - for line in lines.split(':')[1:]: - meta, _, path = line.partition('\x00') - path = path.rstrip('\x00') - a_blob_id, b_blob_id = None, None # Type: Optional[str] - old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4) - # Change type can be R100 - # R: status letter - # 100: score (in case of copy and rename) - change_type = _change_type[0] - score_str = ''.join(_change_type[1:]) - score = int(score_str) if score_str.isdigit() else None - path = path.strip() - a_path = path.encode(defenc) - b_path = path.encode(defenc) - deleted_file = False - new_file = False - copied_file = False - rename_from = None - rename_to = None - - # NOTE: We cannot conclude from the existence of a blob to change type - # as diffs with the working do not have blobs yet - if change_type == 'D': - b_blob_id = None # Optional[str] - deleted_file = True - elif change_type == 'A': - a_blob_id = None - new_file = True - elif change_type == 'C': - copied_file = True - a_path_str, b_path_str = path.split('\x00', 1) - a_path = a_path_str.encode(defenc) - b_path = b_path_str.encode(defenc) - elif change_type == 'R': - a_path_str, b_path_str = path.split('\x00', 1) - a_path = a_path_str.encode(defenc) - b_path = b_path_str.encode(defenc) - rename_from, rename_to = a_path, b_path - elif change_type == 'T': - # Nothing to do - pass - # END add/remove handling - - diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, - new_file, deleted_file, copied_file, rename_from, rename_to, - '', change_type, score) - index.append(diff) - - handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False) + handle_process_output(proc, lambda bytes: cls._handle_diff_line( + bytes, repo, index), None, finalize_process, decode_streams=False) return index @@ -5,14 +5,17 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php """ Module containing all exceptions thrown throughout the git package, """ +from gitdb.exc import BadName # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 from gitdb.exc import * # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 from git.compat import safe_decode # typing ---------------------------------------------------- -from git.repo.base import Repo +from typing import IO, List, Optional, Tuple, Union, TYPE_CHECKING from git.types import PathLike -from typing import IO, List, Optional, Tuple, Union + +if TYPE_CHECKING: + from git.repo.base import Repo # ------------------------------------------------------------------ @@ -63,10 +66,12 @@ class CommandError(GitError): status = "'%s'" % s if isinstance(status, str) else s self._cmd = safe_decode(command[0]) - self._cmdline = ' '.join(str(safe_decode(i)) for i in command) + self._cmdline = ' '.join(safe_decode(i) for i in command) self._cause = status and " due to: %s" % status or "!" - self.stdout = stdout and "\n stdout: '%s'" % safe_decode(str(stdout)) or '' - self.stderr = stderr and "\n stderr: '%s'" % safe_decode(str(stderr)) or '' + stdout_decode = safe_decode(stdout) + stderr_decode = safe_decode(stderr) + self.stdout = stdout_decode and "\n stdout: '%s'" % stdout_decode or '' + self.stderr = stderr_decode and "\n stderr: '%s'" % stderr_decode or '' def __str__(self) -> str: return (self._msg + "\n cmdline: %s%s%s") % ( @@ -142,7 +147,7 @@ class HookExecutionError(CommandError): class RepositoryDirtyError(GitError): """Thrown whenever an operation on a repository fails as it has uncommitted changes that would be overwritten""" - def __init__(self, repo: Repo, message: str) -> None: + def __init__(self, repo: 'Repo', message: str) -> None: self.repo = repo self.message = message diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 23b2416a..897eb98f 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -16,8 +16,8 @@ from .tag import * from .tree import * # Fix import dependency - add IndexObject to the util module, so that it can be # imported by the submodule.base -smutil.IndexObject = IndexObject -smutil.Object = Object +smutil.IndexObject = IndexObject # type: ignore[attr-defined] +smutil.Object = Object # type: ignore[attr-defined] del(smutil) # must come after submodule was made available diff --git a/git/objects/base.py b/git/objects/base.py index cccb5ec6..59f0e836 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -7,6 +7,7 @@ from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex import gitdb.typ as dbtyp import os.path as osp +from typing import Optional # noqa: F401 unused import from .util import get_object_type_by_name @@ -24,7 +25,7 @@ class Object(LazyMixin): TYPES = (dbtyp.str_blob_type, dbtyp.str_tree_type, dbtyp.str_commit_type, dbtyp.str_tag_type) __slots__ = ("repo", "binsha", "size") - type = None # to be set by subclass + type = None # type: Optional[str] # to be set by subclass def __init__(self, repo, binsha): """Initialize an object by identifying it by its binary sha. diff --git a/git/refs/reference.py b/git/refs/reference.py index aaa9b63f..9014f555 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -103,7 +103,7 @@ class Reference(SymbolicReference, LazyMixin, Iterable): #{ Remote Interface - @property + @property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21) @require_remote_ref_path def remote_name(self): """ @@ -114,7 +114,7 @@ class Reference(SymbolicReference, LazyMixin, Iterable): # /refs/remotes/<remote name>/<branch_name> return tokens[2] - @property + @property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21) @require_remote_ref_path def remote_head(self): """:return: Name of the remote head itself, i.e. master. diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index fb9b4f84..22d9c1d5 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -87,7 +87,7 @@ class SymbolicReference(object): """Returns an iterator yielding pairs of sha1/path pairs (as bytes) for the corresponding refs. :note: The packed refs file will be kept open as long as we iterate""" try: - with open(cls._get_packed_refs_path(repo), 'rt') as fp: + with open(cls._get_packed_refs_path(repo), 'rt', encoding='UTF-8') as fp: for line in fp: line = line.strip() if not line: diff --git a/git/remote.py b/git/remote.py index 53349ce7..20b5a551 100644 --- a/git/remote.py +++ b/git/remote.py @@ -9,7 +9,7 @@ import logging import re from git.cmd import handle_process_output, Git -from git.compat import (defenc, force_text, is_win) +from git.compat import (defenc, force_text) from git.exc import GitCommandError from git.util import ( LazyMixin, @@ -36,7 +36,15 @@ from .refs import ( # typing------------------------------------------------------- -from git.repo.Base import Repo +from typing import Any, Optional, Set, TYPE_CHECKING, Union + +from git.types import PathLike + +if TYPE_CHECKING: + from git.repo.base import Repo + from git.objects.commit import Commit + +# ------------------------------------------------------------- log = logging.getLogger('git.remote') log.addHandler(logging.NullHandler()) @@ -47,7 +55,7 @@ __all__ = ('RemoteProgress', 'PushInfo', 'FetchInfo', 'Remote') #{ Utilities -def add_progress(kwargs, git, progress): +def add_progress(kwargs: Any, git: Git, progress: RemoteProgress) -> Any: """Add the --progress flag to the given kwargs dict if supported by the git command. If the actual progress in the given progress instance is not given, we do not request any progress @@ -63,7 +71,7 @@ def add_progress(kwargs, git, progress): #} END utilities -def to_progress_instance(progress): +def to_progress_instance(progress: Optional[RemoteProgress]) -> Union[RemoteProgress, CallableRemoteProgress]: """Given the 'progress' return a suitable object derived from RemoteProgress(). """ @@ -224,7 +232,7 @@ class FetchInfo(object): } @classmethod - def refresh(cls): + def refresh(cls) -> bool: """This gets called by the refresh function (see the top level __init__). """ @@ -247,7 +255,8 @@ class FetchInfo(object): return True - def __init__(self, ref, flags, note='', old_commit=None, remote_ref_path=None): + def __init__(self, ref: SymbolicReference, flags: Set[int], note: str = '', old_commit: Optional[Commit] = None, + remote_ref_path: Optional[PathLike] = None): """ Initialize a new instance """ @@ -257,16 +266,16 @@ class FetchInfo(object): self.old_commit = old_commit self.remote_ref_path = remote_ref_path - def __str__(self): + def __str__(self) -> str: return self.name @property - def name(self): + def name(self) -> str: """:return: Name of our remote ref""" return self.ref.name @property - def commit(self): + def commit(self) -> 'Commit': """:return: Commit of our remote ref""" return self.ref.commit @@ -409,16 +418,6 @@ class Remote(LazyMixin, Iterable): self.repo = repo # type: 'Repo' self.name = name - if is_win: - # some oddity: on windows, python 2.5, it for some reason does not realize - # that it has the config_writer property, but instead calls __getattr__ - # which will not yield the expected results. 'pinging' the members - # with a dir call creates the config_writer property that we require - # ... bugs like these make me wonder whether python really wants to be used - # for production. It doesn't happen on linux though. - dir(self) - # END windows special handling - def __getattr__(self, attr): """Allows to call this instance like remote.special( \\*args, \\*\\*kwargs) to call git-remote special self.name""" diff --git a/git/repo/base.py b/git/repo/base.py index 25363106..ed0a810e 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -4,11 +4,6 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - -from git.objects.tag import TagObject -from git.objects.blob import Blob -from git.objects.tree import Tree -from git.refs.symbolic import SymbolicReference import logging import os import re @@ -30,38 +25,44 @@ from git.index import IndexFile from git.objects import Submodule, RootModule, Commit from git.refs import HEAD, Head, Reference, TagReference from git.remote import Remote, add_progress, to_progress_instance -from git.util import Actor, IterableList, finalize_process, decygpath, hex_to_bin, expand_path +from git.util import Actor, finalize_process, decygpath, hex_to_bin, expand_path, remove_password_if_present import os.path as osp from .fun import rev_parse, is_git_dir, find_submodule_git_dir, touch, find_worktree_git_dir import gc import gitdb -# Typing ------------------------------------------------------------------- - -from typing import (Any, BinaryIO, Callable, Dict, Iterator, List, Mapping, Optional, - TextIO, Tuple, Type, Union, NamedTuple, cast,) -from typing_extensions import Literal -from git.types import PathLike, TBD - -Lit_config_levels = Literal['system', 'global', 'user', 'repository'] - +# typing ------------------------------------------------------ -# -------------------------------------------------------------------------- +from git.compat.typing import Literal +from git.types import TBD, PathLike +from typing import (Any, BinaryIO, Callable, Dict, + Iterator, List, Mapping, Optional, + TextIO, Tuple, Type, Union, + NamedTuple, cast, TYPE_CHECKING) +if TYPE_CHECKING: # only needed for types + from git.util import IterableList + from git.refs.symbolic import SymbolicReference + from git.objects import TagObject, Blob, Tree # NOQA: F401 -class BlameEntry(NamedTuple): - commit: Dict[str, TBD] # Any == 'Commit' type? - linenos: range - orig_path: Optional[str] - orig_linenos: range +Lit_config_levels = Literal['system', 'global', 'user', 'repository'] +# ----------------------------------------------------------- log = logging.getLogger(__name__) __all__ = ('Repo',) +BlameEntry = NamedTuple('BlameEntry', [ + ('commit', Dict[str, TBD]), + ('linenos', range), + ('orig_path', Optional[str]), + ('orig_linenos', range)] +) + + class Repo(object): """Represents a git repository and allows you to query references, gather commit information, generate diffs, create and clone repositories query @@ -221,10 +222,11 @@ class Repo(object): self.git = self.GitCommandWrapperType(self.working_dir) # special handling, in special times - args = [osp.join(self.common_dir, 'objects')] # type: List[Union[str, Git]] + rootpath = osp.join(self.common_dir, 'objects') if issubclass(odbt, GitCmdObjectDB): - args.append(self.git) - self.odb = odbt(*args) + self.odb = odbt(rootpath, self.git) + else: + self.odb = odbt(rootpath) def __enter__(self) -> 'Repo': return self @@ -266,13 +268,14 @@ class Repo(object): # Description property def _get_description(self) -> str: - filename = osp.join(self.git_dir, 'description') if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, 'description') with open(filename, 'rb') as fp: return fp.read().rstrip().decode(defenc) def _set_description(self, descr: str) -> None: - - filename = osp.join(self.git_dir, 'description') if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, 'description') with open(filename, 'wb') as fp: fp.write((descr + '\n').encode(defenc)) @@ -306,7 +309,7 @@ class Repo(object): return self._bare @property - def heads(self) -> IterableList: + def heads(self) -> 'IterableList': """A list of ``Head`` objects representing the branch heads in this repo @@ -314,7 +317,7 @@ class Repo(object): return Head.list_items(self) @property - def references(self) -> IterableList: + def references(self) -> 'IterableList': """A list of Reference objects representing tags, heads and remote references. :return: IterableList(Reference, ...)""" @@ -327,19 +330,19 @@ class Repo(object): branches = heads @property - def index(self) -> IndexFile: + def index(self) -> 'IndexFile': """:return: IndexFile representing this repository's index. :note: This property can be expensive, as the returned ``IndexFile`` will be reinitialized. It's recommended to re-use the object.""" return IndexFile(self) @property - def head(self) -> HEAD: + def head(self) -> 'HEAD': """:return: HEAD Object pointing to the current head reference""" return HEAD(self, 'HEAD') @property - def remotes(self) -> IterableList: + def remotes(self) -> 'IterableList': """A list of Remote objects allowing to access and manipulate remotes :return: ``git.IterableList(Remote, ...)``""" return Remote.list_items(self) @@ -355,13 +358,13 @@ class Repo(object): #{ Submodules @property - def submodules(self) -> IterableList: + def submodules(self) -> 'IterableList': """ :return: git.IterableList(Submodule, ...) of direct submodules available from the current head""" return Submodule.list_items(self) - def submodule(self, name: str) -> IterableList: + def submodule(self, name: str) -> 'IterableList': """ :return: Submodule with the given name :raise ValueError: If no such submodule exists""" try: @@ -393,7 +396,7 @@ class Repo(object): #}END submodules @property - def tags(self) -> IterableList: + def tags(self) -> 'IterableList': """A list of ``Tag`` objects that are available in this repo :return: ``git.IterableList(TagReference, ...)`` """ return TagReference.list_items(self) @@ -405,14 +408,14 @@ class Repo(object): def create_head(self, path: PathLike, commit: str = 'HEAD', force: bool = False, logmsg: Optional[str] = None - ) -> SymbolicReference: + ) -> 'SymbolicReference': """Create a new head within the repository. For more documentation, please see the Head.create method. :return: newly created Head Reference""" return Head.create(self, path, commit, force, logmsg) - def delete_head(self, *heads: HEAD, **kwargs: Any) -> None: + def delete_head(self, *heads: 'SymbolicReference', **kwargs: Any) -> None: """Delete the given heads :param kwargs: Additional keyword arguments to be passed to git-branch""" @@ -458,12 +461,11 @@ class Repo(object): elif config_level == "global": return osp.normpath(osp.expanduser("~/.gitconfig")) elif config_level == "repository": - if self._common_dir: - return osp.normpath(osp.join(self._common_dir, "config")) - elif self.git_dir: - return osp.normpath(osp.join(self.git_dir, "config")) - else: + repo_dir = self._common_dir or self.git_dir + if not repo_dir: raise NotADirectoryError + else: + return osp.normpath(osp.join(repo_dir, "config")) raise ValueError("Invalid configuration level: %r" % config_level) @@ -503,7 +505,8 @@ class Repo(object): repository = configuration file for this repository only""" return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self) - def commit(self, rev: Optional[TBD] = None,) -> Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]: + def commit(self, rev: Optional[TBD] = None + ) -> Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree']: """The Commit object for the specified revision :param rev: revision specifier, see git-rev-parse for viable options. @@ -536,7 +539,7 @@ class Repo(object): return self.rev_parse(str(rev) + "^{tree}") def iter_commits(self, rev: Optional[TBD] = None, paths: Union[PathLike, List[PathLike]] = '', - **kwargs: Any,) -> Iterator[Commit]: + **kwargs: Any) -> Iterator[Commit]: """A list of Commit objects representing the history of a given ref/commit :param rev: @@ -560,8 +563,8 @@ class Repo(object): return Commit.iter_items(self, rev, paths, **kwargs) - def merge_base(self, *rev: TBD, **kwargs: Any, - ) -> List[Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]]: + def merge_base(self, *rev: TBD, **kwargs: Any + ) -> List[Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree', None]]: """Find the closest common ancestor for the given revision (e.g. Commits, Tags, References, etc) :param rev: At least two revs to find the common ancestor for. @@ -574,7 +577,7 @@ class Repo(object): raise ValueError("Please specify at least two revs, got only %i" % len(rev)) # end handle input - res = [] # type: List[Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]] + res = [] # type: List[Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree', None]] try: lines = self.git.merge_base(*rev, **kwargs).splitlines() # List[str] except GitCommandError as err: @@ -608,11 +611,13 @@ class Repo(object): return True def _get_daemon_export(self) -> bool: - filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) return osp.exists(filename) def _set_daemon_export(self, value: object) -> None: - filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) fileexists = osp.exists(filename) if value and not fileexists: touch(filename) @@ -628,7 +633,8 @@ class Repo(object): """The list of alternates for this repo from which objects can be retrieved :return: list of strings being pathnames of alternates""" - alternates_path = osp.join(self.git_dir, 'objects', 'info', 'alternates') if self.git_dir else "" + if self.git_dir: + alternates_path = osp.join(self.git_dir, 'objects', 'info', 'alternates') if osp.exists(alternates_path): with open(alternates_path, 'rb') as f: @@ -768,7 +774,7 @@ class Repo(object): should get a continuous range spanning all line numbers in the file. """ data = self.git.blame(rev, '--', file, p=True, incremental=True, stdout_as_string=False, **kwargs) - commits = {} + commits = {} # type: Dict[str, TBD] stream = (line for line in data.split(b'\n') if line) while True: @@ -776,10 +782,11 @@ class Repo(object): line = next(stream) # when exhausted, causes a StopIteration, terminating this function except StopIteration: return - hexsha, orig_lineno, lineno, num_lines = line.split() - lineno = int(lineno) - num_lines = int(num_lines) - orig_lineno = int(orig_lineno) + split_line = line.split() # type: Tuple[str, str, str, str] + hexsha, orig_lineno_str, lineno_str, num_lines_str = split_line + lineno = int(lineno_str) + num_lines = int(num_lines_str) + orig_lineno = int(orig_lineno_str) if hexsha not in commits: # Now read the next few lines and build up a dict of properties # for this commit @@ -871,12 +878,10 @@ class Repo(object): digits = parts[-1].split(" ") if len(digits) == 3: info = {'id': firstpart} - blames.append([None, [""]]) - elif not info or info['id'] != firstpart: + blames.append([None, []]) + elif info['id'] != firstpart: info = {'id': firstpart} - commits_firstpart = commits.get(firstpart) - blames.append([commits_firstpart, []]) - + blames.append([commits.get(firstpart), []]) # END blame data initialization else: m = self.re_author_committer_start.search(firstpart) @@ -933,8 +938,6 @@ class Repo(object): blames[-1][0] = c if blames[-1][1] is not None: blames[-1][1].append(line) - else: - blames[-1][1] = [line] info = {'id': sha} # END if we collected commit info # END distinguish filename,summary,rest @@ -944,7 +947,7 @@ class Repo(object): @classmethod def init(cls, path: PathLike = None, mkdir: bool = True, odbt: Type[GitCmdObjectDB] = GitCmdObjectDB, - expand_vars: bool = True, **kwargs: Any,) -> 'Repo': + expand_vars: bool = True, **kwargs: Any) -> 'Repo': """Initialize a git repository at the given path if specified :param path: @@ -983,12 +986,8 @@ class Repo(object): @classmethod def _clone(cls, git: 'Git', url: PathLike, path: PathLike, odb_default_type: Type[GitCmdObjectDB], - progress: Optional[Callable], - multi_options: Optional[List[str]] = None, **kwargs: Any, + progress: Optional[Callable], multi_options: Optional[List[str]] = None, **kwargs: Any ) -> 'Repo': - if progress is not None: - progress_checked = to_progress_instance(progress) - odbt = kwargs.pop('odbt', odb_default_type) # when pathlib.Path or other classbased path is passed @@ -1011,13 +1010,16 @@ class Repo(object): if multi_options: multi = ' '.join(multi_options).split(' ') proc = git.clone(multi, Git.polish_url(url), clone_path, with_extended_output=True, as_process=True, - v=True, universal_newlines=True, **add_progress(kwargs, git, progress_checked)) - if progress_checked: - handle_process_output(proc, None, progress_checked.new_message_handler(), + v=True, universal_newlines=True, **add_progress(kwargs, git, progress)) + if progress: + handle_process_output(proc, None, to_progress_instance(progress).new_message_handler(), finalize_process, decode_streams=False) else: (stdout, stderr) = proc.communicate() - log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''), stdout) + cmdline = getattr(proc, 'args', '') + cmdline = remove_password_if_present(cmdline) + + log.debug("Cmd(%s)'s unused stdout: %s", cmdline, stdout) finalize_process(proc, stderr=stderr) # our git command could have a different working dir than our actual @@ -1130,13 +1132,14 @@ class Repo(object): clazz = self.__class__ return '<%s.%s %r>' % (clazz.__module__, clazz.__name__, self.git_dir) - def currently_rebasing_on(self) -> Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]: + def currently_rebasing_on(self) -> Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree', None]: """ :return: The commit which is currently being replayed while rebasing. None if we are not currently rebasing. """ - rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") if self.git_dir else "" + if self.git_dir: + rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") if not osp.isfile(rebase_head_file): return None return self.commit(open(rebase_head_file, "rt").readline().strip()) diff --git a/git/repo/fun.py b/git/repo/fun.py index b8184593..70394081 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -18,11 +18,12 @@ from git.cmd import Git # Typing ---------------------------------------------------------------------- -from .base import Repo -from git.db import GitCmdObjectDB -from git.objects import Commit, TagObject, Blob, Tree -from typing import AnyStr, Union, Optional, cast +from typing import AnyStr, Union, Optional, cast, TYPE_CHECKING from git.types import PathLike +if TYPE_CHECKING: + from .base import Repo + from git.db import GitCmdObjectDB + from git.objects import Commit, TagObject, Blob, Tree # ---------------------------------------------------------------------------- @@ -102,7 +103,7 @@ def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]: return None -def short_to_long(odb: GitCmdObjectDB, hexsha: AnyStr) -> Optional[bytes]: +def short_to_long(odb: 'GitCmdObjectDB', hexsha: AnyStr) -> Optional[bytes]: """:return: long hexadecimal sha1 from the given less-than-40 byte hexsha or None if no candidate could be found. :param hexsha: hexsha with less than 40 byte""" @@ -113,8 +114,8 @@ def short_to_long(odb: GitCmdObjectDB, hexsha: AnyStr) -> Optional[bytes]: # END exception handling -def name_to_object(repo: Repo, name: str, return_ref: bool = False, - ) -> Union[SymbolicReference, Commit, TagObject, Blob, Tree]: +def name_to_object(repo: 'Repo', name: str, return_ref: bool = False + ) -> Union[SymbolicReference, 'Commit', 'TagObject', 'Blob', 'Tree']: """ :return: object specified by the given name, hexshas ( short and long ) as well as references are supported @@ -161,7 +162,7 @@ def name_to_object(repo: Repo, name: str, return_ref: bool = False, return Object.new_from_sha(repo, hex_to_bin(hexsha)) -def deref_tag(tag: Tag) -> TagObject: +def deref_tag(tag: Tag) -> 'TagObject': """Recursively dereference a tag and return the resulting object""" while True: try: @@ -172,7 +173,7 @@ def deref_tag(tag: Tag) -> TagObject: return tag -def to_commit(obj: Object) -> Union[Commit, TagObject]: +def to_commit(obj: Object) -> Union['Commit', 'TagObject']: """Convert the given object to a commit if possible and return it""" if obj.type == 'tag': obj = deref_tag(obj) @@ -183,7 +184,7 @@ def to_commit(obj: Object) -> Union[Commit, TagObject]: return obj -def rev_parse(repo: Repo, rev: str) -> Union[Commit, Tag, Tree, Blob]: +def rev_parse(repo: 'Repo', rev: str) -> Union['Commit', 'Tag', 'Tree', 'Blob']: """ :return: Object at the given revision, either Commit, Tag, Tree or Blob :param rev: git-rev-parse compatible revision specification as string, please see diff --git a/git/types.py b/git/types.py index dc44c123..3e33ae0c 100644 --- a/git/types.py +++ b/git/types.py @@ -1,6 +1,20 @@ -import os # @UnusedImport ## not really unused, is in type string +# -*- coding: utf-8 -*- +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php + +import os +import sys from typing import Union, Any TBD = Any -PathLike = Union[str, 'os.PathLike[str]'] + +if sys.version_info[:2] < (3, 6): + # os.PathLike (PEP-519) only got introduced with Python 3.6 + PathLike = str +elif sys.version_info[:2] < (3, 9): + # Python >= 3.6, < 3.9 + PathLike = Union[str, os.PathLike] +elif sys.version_info[:2] >= (3, 9): + # os.PathLike only becomes subscriptable from Python 3.9 onwards + PathLike = Union[str, os.PathLike[str]] diff --git a/git/util.py b/git/util.py index 2b0c8171..af499028 100644 --- a/git/util.py +++ b/git/util.py @@ -3,7 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -from git.remote import Remote + import contextlib from functools import wraps import getpass @@ -17,11 +17,15 @@ import stat from sys import maxsize import time from unittest import SkipTest +from urllib.parse import urlsplit, urlunsplit # typing --------------------------------------------------------- + from typing import (Any, AnyStr, BinaryIO, Callable, Dict, Generator, IO, List, - NoReturn, Optional, Pattern, Sequence, Tuple, Union, cast) -from git.repo.base import Repo + NoReturn, Optional, Pattern, Sequence, Tuple, Union, cast, TYPE_CHECKING) +if TYPE_CHECKING: + from git.remote import Remote + from git.repo.base import Repo from .types import PathLike, TBD # --------------------------------------------------------------------- @@ -74,7 +78,7 @@ def unbare_repo(func: Callable) -> Callable: encounter a bare repository""" @wraps(func) - def wrapper(self: Remote, *args: Any, **kwargs: Any) -> TBD: + def wrapper(self: 'Remote', *args: Any, **kwargs: Any) -> TBD: if self.repo.bare: raise InvalidGitRepositoryError("Method '%s' cannot operate on bare repositories" % func.__name__) # END bare method @@ -359,6 +363,34 @@ def expand_path(p: PathLike, expand_vars: bool = True) -> Optional[PathLike]: except Exception: return None + +def remove_password_if_present(cmdline): + """ + Parse any command line argument and if on of the element is an URL with a + password, replace it by stars (in-place). + + If nothing found just returns the command line as-is. + + This should be used for every log line that print a command line. + """ + new_cmdline = [] + for index, to_parse in enumerate(cmdline): + new_cmdline.append(to_parse) + try: + url = urlsplit(to_parse) + # Remove password from the URL if present + if url.password is None: + continue + + edited_url = url._replace( + netloc=url.netloc.replace(url.password, "*****")) + new_cmdline[index] = urlunsplit(edited_url) + except ValueError: + # This is not a valid URL + continue + return new_cmdline + + #} END utilities #{ Classes @@ -686,7 +718,7 @@ class Stats(object): self.files = files @classmethod - def _list_from_string(cls, repo: Repo, text: str) -> 'Stats': + def _list_from_string(cls, repo: 'Repo', text: str) -> 'Stats': """Create a Stat object from output retrieved by git-diff. :return: git.Stat""" @@ -924,6 +956,7 @@ class IterableList(list): def __delitem__(self, index: Union[int, str, slice]) -> None: + delindex = cast(int, index) if not isinstance(index, int): delindex = -1 assert not isinstance(index, slice) @@ -949,7 +982,7 @@ class Iterable(object): _id_attribute_ = "attribute that most suitably identifies your instance" @classmethod - def list_items(cls, repo: Repo, *args: Any, **kwargs: Any) -> 'IterableList': + def list_items(cls, repo: 'Repo', *args: Any, **kwargs: Any) -> 'IterableList': """ Find all items of this type - subclasses can specify args and kwargs differently. If no args are given, subclasses are obliged to return all items if no additional @@ -963,7 +996,7 @@ class Iterable(object): return out_list @classmethod - def iter_items(cls, repo: Repo, *args: Any, **kwargs: Any) -> NoReturn: + def iter_items(cls, repo: 'Repo', *args: Any, **kwargs: Any) -> NoReturn: """For more information about the arguments, see list_items :return: iterator yielding Items""" raise NotImplementedError("To be implemented by Subclass") @@ -1,6 +1,9 @@ [mypy] -disallow_untyped_defs = True +# TODO: enable when we've fully annotated everything +#disallow_untyped_defs = True -mypy_path = 'git' +# TODO: remove when 'gitdb' is fully annotated +[mypy-gitdb.*] +ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt index c4e8340d..d980f668 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ gitdb>=4.0.1,<5 +typing-extensions>=3.7.4.0;python_version<"3.8" diff --git a/test-requirements.txt b/test-requirements.txt index abda95cf..e06d2be1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,3 +4,5 @@ flake8 tox virtualenv nose +gitdb>=4.0.1,<5 +typing-extensions>=3.7.4.0;python_version<"3.8" diff --git a/test/fixtures/diff_file_with_colon b/test/fixtures/diff_file_with_colon Binary files differnew file mode 100644 index 00000000..4058b171 --- /dev/null +++ b/test/fixtures/diff_file_with_colon diff --git a/test/test_clone.py b/test/test_clone.py new file mode 100644 index 00000000..e9f6714d --- /dev/null +++ b/test/test_clone.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php + +from pathlib import Path +import re + +import git + +from .lib import ( + TestBase, + with_rw_directory, +) + + +class TestClone(TestBase): + @with_rw_directory + def test_checkout_in_non_empty_dir(self, rw_dir): + non_empty_dir = Path(rw_dir) + garbage_file = non_empty_dir / 'not-empty' + garbage_file.write_text('Garbage!') + + # Verify that cloning into the non-empty dir fails while complaining about + # the target directory not being empty/non-existent + try: + self.rorepo.clone(non_empty_dir) + except git.GitCommandError as exc: + self.assertTrue(exc.stderr, "GitCommandError's 'stderr' is unexpectedly empty") + expr = re.compile(r'(?is).*\bfatal:\s+destination\s+path\b.*\bexists\b.*\bnot\b.*\bempty\s+directory\b') + self.assertTrue(expr.search(exc.stderr), '"%s" does not match "%s"' % (expr.pattern, exc.stderr)) + else: + self.fail("GitCommandError not raised") diff --git a/test/test_diff.py b/test/test_diff.py index c6c9b67a..9b20893a 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -7,6 +7,7 @@ import ddt import shutil import tempfile +import unittest from git import ( Repo, GitCommandError, @@ -220,6 +221,12 @@ class TestDiff(TestBase): self.assertIsNotNone(res[0].deleted_file) self.assertIsNone(res[0].b_path,) + @unittest.skip("This currently fails and would need someone to improve diff parsing") + def test_diff_file_with_colon(self): + output = fixture('diff_file_with_colon') + res = [] + Diff._handle_diff_line(output, None, res) + def test_diff_initial_commit(self): initial_commit = self.rorepo.commit('33ebe7acec14b25c5f84f35a664803fcab2f7781') diff --git a/test/test_repo.py b/test/test_repo.py index d5ea8664..8dc17833 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -238,6 +238,21 @@ class TestRepo(TestBase): except UnicodeEncodeError: self.fail('Raised UnicodeEncodeError') + @with_rw_directory + def test_leaking_password_in_clone_logs(self, rw_dir): + password = "fakepassword1234" + try: + Repo.clone_from( + url="https://fakeuser:{}@fakerepo.example.com/testrepo".format( + password), + to_path=rw_dir) + except GitCommandError as err: + assert password not in str(err), "The error message '%s' should not contain the password" % err + # Working example from a blank private project + Repo.clone_from( + url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", + to_path=rw_dir) + @with_rw_repo('HEAD') def test_max_chunk_size(self, repo): class TestOutputStream(TestBase): diff --git a/test/test_util.py b/test/test_util.py index 5eba6c50..ddc5f628 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -30,7 +30,8 @@ from git.util import ( Actor, IterableList, cygpath, - decygpath + decygpath, + remove_password_if_present, ) @@ -322,3 +323,20 @@ class TestUtils(TestBase): t2 = pickle.loads(pickle.dumps(t1)) self.assertEqual(t1._offset, t2._offset) self.assertEqual(t1._name, t2._name) + + def test_remove_password_from_command_line(self): + password = "fakepassword1234" + url_with_pass = "https://fakeuser:{}@fakerepo.example.com/testrepo".format(password) + url_without_pass = "https://fakerepo.example.com/testrepo" + + cmd_1 = ["git", "clone", "-v", url_with_pass] + cmd_2 = ["git", "clone", "-v", url_without_pass] + cmd_3 = ["no", "url", "in", "this", "one"] + + redacted_cmd_1 = remove_password_if_present(cmd_1) + assert password not in " ".join(redacted_cmd_1) + # Check that we use a copy + assert cmd_1 is not redacted_cmd_1 + assert password in " ".join(cmd_1) + assert cmd_2 == remove_password_if_present(cmd_2) + assert cmd_3 == remove_password_if_present(cmd_3) @@ -14,6 +14,14 @@ commands = coverage run --omit="git/test/*" -m unittest --buffer {posargs} [testenv:flake8] commands = flake8 --ignore=W293,E265,E266,W503,W504,E731 {posargs} +[testenv:type] +description = type check ourselves +deps = + {[testenv]deps} + mypy +commands = + mypy -p git + [testenv:venv] commands = {posargs} @@ -23,6 +31,7 @@ commands = {posargs} # E266 = too many leading '#' for block comment # E731 = do not assign a lambda expression, use a def # W293 = Blank line contains whitespace -ignore = E265,W293,E266,E731 +# W504 = Line break after operator +ignore = E265,W293,E266,E731, W504 max-line-length = 120 exclude = .tox,.venv,build,dist,doc,git/ext/ |