From 6752fad0e93d1d2747f56be30a52fea212bd15d6 Mon Sep 17 00:00:00 2001 From: yobmod Date: Mon, 3 May 2021 15:59:07 +0100 Subject: [PATCH] add initial types to remote.py --- .github/FUNDING.yml | 1 + CONTRIBUTING.md | 3 +- MANIFEST.in | 7 +- Makefile | 4 +- VERSION | 2 +- doc/source/changes.rst | 13 ++- git/__init__.py | 9 +- git/cmd.py | 56 +++++----- git/{compat.py => compat/__init__.py} | 39 ++++++- git/compat/typing.py | 13 +++ git/config.py | 7 +- git/db.py | 17 +-- git/diff.py | 126 ++++++++++----------- git/exc.py | 17 ++- git/objects/__init__.py | 4 +- git/objects/base.py | 3 +- git/refs/reference.py | 4 +- git/refs/symbolic.py | 2 +- git/remote.py | 37 +++---- git/repo/base.py | 151 +++++++++++++------------- git/repo/fun.py | 21 ++-- git/types.py | 18 ++- git/util.py | 47 ++++++-- mypy.ini | 7 +- requirements.txt | 1 + test-requirements.txt | 2 + test/fixtures/diff_file_with_colon | Bin 0 -> 351 bytes test/test_clone.py | 32 ++++++ test/test_diff.py | 7 ++ test/test_repo.py | 15 +++ test/test_util.py | 20 +++- tox.ini | 11 +- 32 files changed, 453 insertions(+), 243 deletions(-) create mode 100644 .github/FUNDING.yml rename git/{compat.py => compat/__init__.py} (75%) create mode 100644 git/compat/typing.py create mode 100644 test/fixtures/diff_file_with_colon create mode 100644 test/test_clone.py diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..80819f5d8 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: byron diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4217cbaf9..f685e7e72 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,6 +5,7 @@ The following is a short step-by-step rundown of what one typically would do to * [fork this project](https://github.com/gitpython-developers/GitPython/fork) on GitHub. * For setting up the environment to run the self tests, please look at `.travis.yml`. * Please try to **write a test that fails unless the contribution is present.** -* Feel free to add yourself to AUTHORS file. +* Try to avoid massive commits and prefer to take small steps, with one commit for each. +* Feel free to add yourself to AUTHORS file. * Create a pull request. diff --git a/MANIFEST.in b/MANIFEST.in index 5fd771db3..f02721fc6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,11 @@ -include VERSION -include LICENSE -include CHANGES include AUTHORS +include CHANGES include CONTRIBUTING.md +include LICENSE include README.md +include VERSION include requirements.txt +include test-requirements.txt recursive-include doc * recursive-exclude test * diff --git a/Makefile b/Makefile index 709813ff2..f5d8a1089 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ release: clean make force_release force_release: clean - git push --tags origin master + git push --tags origin main python3 setup.py sdist bdist_wheel twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* @@ -24,7 +24,7 @@ docker-build: test: docker-build # NOTE!!! - # NOTE!!! If you are not running from master or have local changes then tests will fail + # NOTE!!! If you are not running from main or have local changes then tests will fail # NOTE!!! docker run --rm -v ${CURDIR}:/src -w /src -t gitpython:xenial tox diff --git a/VERSION b/VERSION index 55f20a1a9..b5f785d2d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.13 +3.1.15 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 405179d0c..1b916f30f 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,7 +2,15 @@ Changelog ========= -3.1.?? +3.1.15 +====== + +* add deprectation warning for python 3.5 + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/47?closed=1 + +3.1.14 ====== * git.Commit objects now have a ``replace`` method that will return a @@ -10,6 +18,9 @@ Changelog * Add python 3.9 support * Drop python 3.4 support +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/46?closed=1 + 3.1.13 ====== diff --git a/git/__init__.py b/git/__init__.py index e2f960db7..ae9254a26 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -5,6 +5,7 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php # flake8: noqa #@PydevCodeAnalysisIgnore +from git.exc import * # @NoMove @IgnorePep8 import inspect import os import sys @@ -16,8 +17,6 @@ __version__ = 'git' - - #{ Initialization def _init_externals() -> None: """Initialize external projects by putting them into the path""" @@ -32,13 +31,13 @@ def _init_externals() -> None: #} END initialization + ################# _init_externals() ################# #{ Imports -from git.exc import * # @NoMove @IgnorePep8 try: from git.config import GitConfigParser # @NoMove @IgnorePep8 from git.objects import * # @NoMove @IgnorePep8 @@ -68,7 +67,8 @@ def _init_externals() -> None: #{ Initialize git executable path GIT_OK = None -def refresh(path:Optional[PathLike]=None) -> None: + +def refresh(path: Optional[PathLike] = None) -> None: """Convenience method for setting the git executable path.""" global GIT_OK GIT_OK = False @@ -81,6 +81,7 @@ def refresh(path:Optional[PathLike]=None) -> None: GIT_OK = True #} END initialize git executable path + ################# try: refresh() diff --git a/git/cmd.py b/git/cmd.py index bac162176..ac3ca2ec1 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -19,7 +19,7 @@ import threading from collections import OrderedDict from textwrap import dedent -from typing import Any, Dict, List, Optional +import warnings from git.compat import ( defenc, @@ -29,7 +29,7 @@ is_win, ) from git.exc import CommandError -from git.util import is_cygwin_git, cygpath, expand_path +from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present from .exc import ( GitCommandError, @@ -40,8 +40,6 @@ stream_copy, ) -from .types import PathLike - execute_kwargs = {'istream', 'with_extended_output', 'with_exceptions', 'as_process', 'stdout_as_string', 'output_stream', 'with_stdout', 'kill_after_timeout', @@ -85,8 +83,8 @@ def pump_stream(cmdline, name, stream, is_decode, handler): line = line.decode(defenc) handler(line) except Exception as ex: - log.error("Pumping %r of cmd(%s) failed due to: %r", name, cmdline, ex) - raise CommandError(['<%s-pump>' % name] + cmdline, ex) from ex + log.error("Pumping %r of cmd(%s) failed due to: %r", name, remove_password_if_present(cmdline), ex) + raise CommandError(['<%s-pump>' % name] + remove_password_if_present(cmdline), ex) from ex finally: stream.close() @@ -105,7 +103,7 @@ def pump_stream(cmdline, name, stream, is_decode, handler): for name, stream, handler in pumps: t = threading.Thread(target=pump_stream, args=(cmdline, name, stream, decode_streams, handler)) - t.setDaemon(True) + t.daemon = True t.start() threads.append(t) @@ -140,7 +138,7 @@ def dict_to_slots_and__excluded_are_none(self, d, excluded=()): ## CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards, # see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal -PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP +PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined] if is_win else 0) @@ -212,7 +210,7 @@ def refresh(cls, path=None): # - a GitCommandNotFound error is spawned by ourselves # - a PermissionError is spawned if the git executable provided # cannot be executed for whatever reason - + has_git = False try: cls().version() @@ -408,7 +406,7 @@ def read_all_from_possibly_closed_stream(stream): if status != 0: errstr = read_all_from_possibly_closed_stream(self.proc.stderr) log.debug('AutoInterrupt wait stderr: %r' % (errstr,)) - raise GitCommandError(self.args, status, errstr) + raise GitCommandError(remove_password_if_present(self.args), status, errstr) # END status handling return status # END auto interrupt @@ -500,7 +498,7 @@ def readlines(self, size=-1): # skipcq: PYL-E0301 def __iter__(self): return self - + def __next__(self): return self.next() @@ -519,7 +517,7 @@ def __del__(self): self._stream.read(bytes_left + 1) # END handle incomplete read - def __init__(self, working_dir: Optional[PathLike]=None) -> None: + def __init__(self, working_dir=None): """Initialize this instance with: :param working_dir: @@ -528,12 +526,12 @@ def __init__(self, working_dir: Optional[PathLike]=None) -> None: It is meant to be the working tree directory if available, or the .git directory in case of bare repositories.""" super(Git, self).__init__() - self._working_dir = expand_path(working_dir) if working_dir is not None else None + self._working_dir = expand_path(working_dir) self._git_options = () - self._persistent_git_options = [] # type: List[str] + self._persistent_git_options = [] # Extra environment variables to pass to git commands - self._environment = {} # type: Dict[str, Any] + self._environment = {} # cached command slots self.cat_file_header = None @@ -547,7 +545,7 @@ def __getattr__(self, name): return LazyMixin.__getattr__(self, name) return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) - def set_persistent_git_options(self, **kwargs) -> None: + def set_persistent_git_options(self, **kwargs): """Specify command line options to the git executable for subsequent subcommand calls @@ -641,7 +639,7 @@ def execute(self, command, :param env: A dictionary of environment variables to be passed to `subprocess.Popen`. - + :param max_chunk_size: Maximum number of bytes in one chunk of data passed to the output_stream in one invocation of write() method. If the given number is not positive then @@ -685,8 +683,10 @@ def execute(self, command, :note: If you add additional keyword arguments to the signature of this method, you must update the execute_kwargs tuple housed in this module.""" + # Remove password for the command if present + redacted_command = remove_password_if_present(command) if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != 'full' or as_process): - log.info(' '.join(command)) + log.info(' '.join(redacted_command)) # Allow the user to have the command executed in their working dir. cwd = self._working_dir or os.getcwd() @@ -707,7 +707,7 @@ def execute(self, command, if is_win: cmd_not_found_exception = OSError if kill_after_timeout: - raise GitCommandError(command, '"kill_after_timeout" feature is not supported on Windows.') + raise GitCommandError(redacted_command, '"kill_after_timeout" feature is not supported on Windows.') else: if sys.version_info[0] > 2: cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable @@ -722,7 +722,7 @@ def execute(self, command, if istream: istream_ok = "" log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", - command, cwd, universal_newlines, shell, istream_ok) + redacted_command, cwd, universal_newlines, shell, istream_ok) try: proc = Popen(command, env=env, @@ -738,7 +738,7 @@ def execute(self, command, **subprocess_kwargs ) except cmd_not_found_exception as err: - raise GitCommandNotFound(command, err) from err + raise GitCommandNotFound(redacted_command, err) from err if as_process: return self.AutoInterrupt(proc, command) @@ -788,7 +788,7 @@ def _kill_process(pid): watchdog.cancel() if kill_check.isSet(): stderr_value = ('Timeout: the command "%s" did not complete in %d ' - 'secs.' % (" ".join(command), kill_after_timeout)) + 'secs.' % (" ".join(redacted_command), kill_after_timeout)) if not universal_newlines: stderr_value = stderr_value.encode(defenc) # strip trailing "\n" @@ -812,7 +812,7 @@ def _kill_process(pid): proc.stderr.close() if self.GIT_PYTHON_TRACE == 'full': - cmdstr = " ".join(command) + cmdstr = " ".join(redacted_command) def as_text(stdout_value): return not output_stream and safe_decode(stdout_value) or '' @@ -828,7 +828,7 @@ def as_text(stdout_value): # END handle debug printing if with_exceptions and status != 0: - raise GitCommandError(command, status, stderr_value, stdout_value) + raise GitCommandError(redacted_command, status, stderr_value, stdout_value) if isinstance(stdout_value, bytes) and stdout_as_string: # could also be output_stream stdout_value = safe_decode(stdout_value) @@ -905,8 +905,14 @@ def transform_kwarg(self, name, value, split_single_char_options): def transform_kwargs(self, split_single_char_options=True, **kwargs): """Transforms Python style kwargs into git command line options.""" + # Python 3.6 preserves the order of kwargs and thus has a stable + # order. For older versions sort the kwargs by the key to get a stable + # order. + if sys.version_info[:2] < (3, 6): + kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0])) + warnings.warn("Python 3.5 support is deprecated and will be removed 2021-09-05.\n" + + "It does not preserve the order for key-word arguments and enforce lexical sorting instead.") args = [] - kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0])) for k, v in kwargs.items(): if isinstance(v, (list, tuple)): for value in v: diff --git a/git/compat.py b/git/compat/__init__.py similarity index 75% rename from git/compat.py rename to git/compat/__init__.py index 4fe394ae0..c4bd2aa36 100644 --- a/git/compat.py +++ b/git/compat/__init__.py @@ -16,9 +16,22 @@ force_text # @UnusedImport ) -from typing import Any, AnyStr, Dict, Optional, Type +# typing -------------------------------------------------------------------- + +from typing import ( + Any, + AnyStr, + Dict, + IO, + Optional, + Type, + Union, + overload, +) from git.types import TBD +# --------------------------------------------------------------------------- + is_win = (os.name == 'nt') # type: bool is_posix = (os.name == 'posix') @@ -26,7 +39,13 @@ defenc = sys.getfilesystemencoding() -def safe_decode(s: Optional[AnyStr]) -> Optional[str]: +@overload +def safe_decode(s: None) -> None: ... + +@overload +def safe_decode(s: Union[IO[str], AnyStr]) -> str: ... + +def safe_decode(s: Union[IO[str], AnyStr, None]) -> Optional[str]: """Safely decodes a binary string to unicode""" if isinstance(s, str): return s @@ -38,6 +57,11 @@ def safe_decode(s: Optional[AnyStr]) -> Optional[str]: raise TypeError('Expected bytes or text, but got %r' % (s,)) +@overload +def safe_encode(s: None) -> None: ... + +@overload +def safe_encode(s: AnyStr) -> bytes: ... def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: """Safely encodes a binary string to unicode""" @@ -51,6 +75,12 @@ def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: raise TypeError('Expected bytes or text, but got %r' % (s,)) +@overload +def win_encode(s: None) -> None: ... + +@overload +def win_encode(s: AnyStr) -> bytes: ... + def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: """Encode unicodes for process arguments on Windows.""" if isinstance(s, str): @@ -62,9 +92,9 @@ def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: return None -def with_metaclass(meta: Type[Any], *bases: Any) -> 'metaclass': # type: ignore ## mypy cannot understand dynamic class creation +def with_metaclass(meta: Type[Any], *bases: Any) -> 'metaclass': # type: ignore ## mypy cannot understand dynamic class creation """copied from https://github.com/Byron/bcore/blob/master/src/python/butility/future.py#L15""" - + class metaclass(meta): # type: ignore __call__ = type.__call__ __init__ = type.__init__ # type: ignore @@ -75,4 +105,3 @@ def __new__(cls, name: str, nbases: Optional[int], d: Dict[str, Any]) -> TBD: return meta(name, bases, d) return metaclass(meta.__name__ + 'Helper', None, {}) - diff --git a/git/compat/typing.py b/git/compat/typing.py new file mode 100644 index 000000000..925c5ba2e --- /dev/null +++ b/git/compat/typing.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# config.py +# Copyright (C) 2021 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php + +import sys + +if sys.version_info[:2] >= (3, 8): + from typing import Final, Literal # noqa: F401 +else: + from typing_extensions import Final, Literal # noqa: F401 diff --git a/git/config.py b/git/config.py index ffbbfab40..0c8d975db 100644 --- a/git/config.py +++ b/git/config.py @@ -16,14 +16,13 @@ import fnmatch from collections import OrderedDict -from typing_extensions import Literal - from git.compat import ( defenc, force_text, with_metaclass, is_win, ) +from git.compat.typing import Literal from git.util import LockFile import os.path as osp @@ -196,7 +195,7 @@ def items_all(self): return [(k, self.getall(k)) for k in self] -def get_config_path(config_level: Literal['system','global','user','repository']) -> str: +def get_config_path(config_level: Literal['system', 'global', 'user', 'repository']) -> str: # we do not support an absolute path of the gitconfig on windows , # use the global config instead @@ -216,7 +215,7 @@ def get_config_path(config_level: Literal['system','global','user','repository'] raise ValueError("Invalid configuration level: %r" % config_level) -class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): +class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): # type: ignore ## mypy does not understand dynamic class creation # noqa: E501 """Implements specifics required to read git style configuration files. diff --git a/git/db.py b/git/db.py index ef2b0b2ef..dc60c5552 100644 --- a/git/db.py +++ b/git/db.py @@ -1,5 +1,4 @@ """Module with our own gitdb implementation - it uses the git command""" -from typing import AnyStr from git.util import bin_to_hex, hex_to_bin from gitdb.base import ( OInfo, @@ -7,21 +6,23 @@ ) from gitdb.db import GitDB # @UnusedImport from gitdb.db import LooseObjectDB -from gitdb.exc import BadObject -from .exc import GitCommandError +from gitdb.exc import BadObject +from git.exc import GitCommandError # typing------------------------------------------------- -from .cmd import Git -from .types import PathLike +from typing import TYPE_CHECKING, AnyStr +from git.types import PathLike + +if TYPE_CHECKING: + from git.cmd import Git + # -------------------------------------------------------- __all__ = ('GitCmdObjectDB', 'GitDB') -# class GitCmdObjectDB(CompoundDB, ObjectDBW): - class GitCmdObjectDB(LooseObjectDB): @@ -33,7 +34,7 @@ class GitCmdObjectDB(LooseObjectDB): have packs and the other implementations """ - def __init__(self, root_path: PathLike, git: Git) -> None: + def __init__(self, root_path: PathLike, git: 'Git') -> None: """Initialize this instance with the root and a git command""" super(GitCmdObjectDB, self).__init__(root_path) self._git = git diff --git a/git/diff.py b/git/diff.py index b25aadc76..943916ea8 100644 --- a/git/diff.py +++ b/git/diff.py @@ -15,11 +15,14 @@ # typing ------------------------------------------------------------------ -from .objects.tree import Tree -from git.repo.base import Repo -from typing_extensions import Final, Literal +from typing import Any, Iterator, List, Match, Optional, Tuple, Type, Union, TYPE_CHECKING +from git.compat.typing import Final, Literal from git.types import TBD -from typing import Any, Iterator, List, Match, Optional, Tuple, Type, Union + +if TYPE_CHECKING: + from .objects.tree import Tree + from git.repo.base import Repo + Lit_change_type = Literal['A', 'D', 'M', 'R', 'T'] # ------------------------------------------------------------------------ @@ -27,7 +30,7 @@ __all__ = ('Diffable', 'DiffIndex', 'Diff', 'NULL_TREE') # Special object to compare against the empty tree in diffs -NULL_TREE: Final[object] = object() +NULL_TREE = object() # type: Final[object] _octal_byte_re = re.compile(b'\\\\([0-9]{3})') @@ -79,7 +82,7 @@ def _process_diff_args(self, args: List[Union[str, 'Diffable', object]]) -> List Subclasses can use it to alter the behaviour of the superclass""" return args - def diff(self, other: Union[Type[Index], Type[Tree], object, None, str] = Index, + def diff(self, other: Union[Type[Index], Type['Tree'], object, None, str] = Index, paths: Union[str, List[str], Tuple[str, ...], None] = None, create_patch: bool = False, **kwargs: Any) -> 'DiffIndex': """Creates diffs between two items being trees, trees and index or an @@ -271,7 +274,7 @@ class Diff(object): "new_file", "deleted_file", "copied_file", "raw_rename_from", "raw_rename_to", "diff", "change_type", "score") - def __init__(self, repo: Repo, + def __init__(self, repo: 'Repo', a_rawpath: Optional[bytes], b_rawpath: Optional[bytes], a_blob_id: Union[str, bytes, None], b_blob_id: Union[str, bytes, None], a_mode: Union[bytes, str, None], b_mode: Union[bytes, str, None], @@ -425,7 +428,7 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m return None @classmethod - def _index_from_patch_format(cls, repo: Repo, proc: TBD) -> DiffIndex: + def _index_from_patch_format(cls, repo: 'Repo', proc: TBD) -> DiffIndex: """Create a new DiffIndex from the given text which must be in patch format :param repo: is the repository we are operating on - it is required :param stream: result of 'git diff' as a stream (supporting file protocol) @@ -487,6 +490,58 @@ def _index_from_patch_format(cls, repo: Repo, proc: TBD) -> DiffIndex: return index + @staticmethod + def _handle_diff_line(lines_bytes: bytes, repo: 'Repo', index: TBD) -> None: + lines = lines_bytes.decode(defenc) + + for line in lines.split(':')[1:]: + meta, _, path = line.partition('\x00') + path = path.rstrip('\x00') + a_blob_id, b_blob_id = None, None # Type: Optional[str] + old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4) + # Change type can be R100 + # R: status letter + # 100: score (in case of copy and rename) + change_type = _change_type[0] + score_str = ''.join(_change_type[1:]) + score = int(score_str) if score_str.isdigit() else None + path = path.strip() + a_path = path.encode(defenc) + b_path = path.encode(defenc) + deleted_file = False + new_file = False + copied_file = False + rename_from = None + rename_to = None + + # NOTE: We cannot conclude from the existence of a blob to change type + # as diffs with the working do not have blobs yet + if change_type == 'D': + b_blob_id = None # Optional[str] + deleted_file = True + elif change_type == 'A': + a_blob_id = None + new_file = True + elif change_type == 'C': + copied_file = True + a_path_str, b_path_str = path.split('\x00', 1) + a_path = a_path_str.encode(defenc) + b_path = b_path_str.encode(defenc) + elif change_type == 'R': + a_path_str, b_path_str = path.split('\x00', 1) + a_path = a_path_str.encode(defenc) + b_path = b_path_str.encode(defenc) + rename_from, rename_to = a_path, b_path + elif change_type == 'T': + # Nothing to do + pass + # END add/remove handling + + diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, + new_file, deleted_file, copied_file, rename_from, rename_to, + '', change_type, score) + index.append(diff) + @classmethod def _index_from_raw_format(cls, repo: 'Repo', proc: TBD) -> DiffIndex: """Create a new DiffIndex from the given stream which must be in raw format. @@ -495,58 +550,7 @@ def _index_from_raw_format(cls, repo: 'Repo', proc: TBD) -> DiffIndex: # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() - - def handle_diff_line(lines_bytes: bytes) -> None: - lines = lines_bytes.decode(defenc) - - for line in lines.split(':')[1:]: - meta, _, path = line.partition('\x00') - path = path.rstrip('\x00') - a_blob_id, b_blob_id = None, None # Type: Optional[str] - old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4) - # Change type can be R100 - # R: status letter - # 100: score (in case of copy and rename) - change_type = _change_type[0] - score_str = ''.join(_change_type[1:]) - score = int(score_str) if score_str.isdigit() else None - path = path.strip() - a_path = path.encode(defenc) - b_path = path.encode(defenc) - deleted_file = False - new_file = False - copied_file = False - rename_from = None - rename_to = None - - # NOTE: We cannot conclude from the existence of a blob to change type - # as diffs with the working do not have blobs yet - if change_type == 'D': - b_blob_id = None # Optional[str] - deleted_file = True - elif change_type == 'A': - a_blob_id = None - new_file = True - elif change_type == 'C': - copied_file = True - a_path_str, b_path_str = path.split('\x00', 1) - a_path = a_path_str.encode(defenc) - b_path = b_path_str.encode(defenc) - elif change_type == 'R': - a_path_str, b_path_str = path.split('\x00', 1) - a_path = a_path_str.encode(defenc) - b_path = b_path_str.encode(defenc) - rename_from, rename_to = a_path, b_path - elif change_type == 'T': - # Nothing to do - pass - # END add/remove handling - - diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, - new_file, deleted_file, copied_file, rename_from, rename_to, - '', change_type, score) - index.append(diff) - - handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False) + handle_process_output(proc, lambda bytes: cls._handle_diff_line( + bytes, repo, index), None, finalize_process, decode_streams=False) return index diff --git a/git/exc.py b/git/exc.py index c02b2b3a3..6e646921c 100644 --- a/git/exc.py +++ b/git/exc.py @@ -5,14 +5,17 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php """ Module containing all exceptions thrown throughout the git package, """ +from gitdb.exc import BadName # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 from gitdb.exc import * # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 from git.compat import safe_decode # typing ---------------------------------------------------- -from git.repo.base import Repo +from typing import IO, List, Optional, Tuple, Union, TYPE_CHECKING from git.types import PathLike -from typing import IO, List, Optional, Tuple, Union + +if TYPE_CHECKING: + from git.repo.base import Repo # ------------------------------------------------------------------ @@ -63,10 +66,12 @@ def __init__(self, command: Union[List[str], Tuple[str, ...], str], status = "'%s'" % s if isinstance(status, str) else s self._cmd = safe_decode(command[0]) - self._cmdline = ' '.join(str(safe_decode(i)) for i in command) + self._cmdline = ' '.join(safe_decode(i) for i in command) self._cause = status and " due to: %s" % status or "!" - self.stdout = stdout and "\n stdout: '%s'" % safe_decode(str(stdout)) or '' - self.stderr = stderr and "\n stderr: '%s'" % safe_decode(str(stderr)) or '' + stdout_decode = safe_decode(stdout) + stderr_decode = safe_decode(stderr) + self.stdout = stdout_decode and "\n stdout: '%s'" % stdout_decode or '' + self.stderr = stderr_decode and "\n stderr: '%s'" % stderr_decode or '' def __str__(self) -> str: return (self._msg + "\n cmdline: %s%s%s") % ( @@ -142,7 +147,7 @@ def __init__(self, command: Union[List[str], Tuple[str, ...], str], status: Opti class RepositoryDirtyError(GitError): """Thrown whenever an operation on a repository fails as it has uncommitted changes that would be overwritten""" - def __init__(self, repo: Repo, message: str) -> None: + def __init__(self, repo: 'Repo', message: str) -> None: self.repo = repo self.message = message diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 23b2416ae..897eb98fa 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -16,8 +16,8 @@ from .tree import * # Fix import dependency - add IndexObject to the util module, so that it can be # imported by the submodule.base -smutil.IndexObject = IndexObject -smutil.Object = Object +smutil.IndexObject = IndexObject # type: ignore[attr-defined] +smutil.Object = Object # type: ignore[attr-defined] del(smutil) # must come after submodule was made available diff --git a/git/objects/base.py b/git/objects/base.py index cccb5ec66..59f0e8368 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -7,6 +7,7 @@ import gitdb.typ as dbtyp import os.path as osp +from typing import Optional # noqa: F401 unused import from .util import get_object_type_by_name @@ -24,7 +25,7 @@ class Object(LazyMixin): TYPES = (dbtyp.str_blob_type, dbtyp.str_tree_type, dbtyp.str_commit_type, dbtyp.str_tag_type) __slots__ = ("repo", "binsha", "size") - type = None # to be set by subclass + type = None # type: Optional[str] # to be set by subclass def __init__(self, repo, binsha): """Initialize an object by identifying it by its binary sha. diff --git a/git/refs/reference.py b/git/refs/reference.py index aaa9b63fe..9014f5558 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -103,7 +103,7 @@ def iter_items(cls, repo, common_path=None): #{ Remote Interface - @property + @property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21) @require_remote_ref_path def remote_name(self): """ @@ -114,7 +114,7 @@ def remote_name(self): # /refs/remotes// return tokens[2] - @property + @property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21) @require_remote_ref_path def remote_head(self): """:return: Name of the remote head itself, i.e. master. diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index fb9b4f84b..22d9c1d51 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -87,7 +87,7 @@ def _iter_packed_refs(cls, repo): """Returns an iterator yielding pairs of sha1/path pairs (as bytes) for the corresponding refs. :note: The packed refs file will be kept open as long as we iterate""" try: - with open(cls._get_packed_refs_path(repo), 'rt') as fp: + with open(cls._get_packed_refs_path(repo), 'rt', encoding='UTF-8') as fp: for line in fp: line = line.strip() if not line: diff --git a/git/remote.py b/git/remote.py index 53349ce70..20b5a5514 100644 --- a/git/remote.py +++ b/git/remote.py @@ -9,7 +9,7 @@ import re from git.cmd import handle_process_output, Git -from git.compat import (defenc, force_text, is_win) +from git.compat import (defenc, force_text) from git.exc import GitCommandError from git.util import ( LazyMixin, @@ -36,7 +36,15 @@ # typing------------------------------------------------------- -from git.repo.Base import Repo +from typing import Any, Optional, Set, TYPE_CHECKING, Union + +from git.types import PathLike + +if TYPE_CHECKING: + from git.repo.base import Repo + from git.objects.commit import Commit + +# ------------------------------------------------------------- log = logging.getLogger('git.remote') log.addHandler(logging.NullHandler()) @@ -47,7 +55,7 @@ #{ Utilities -def add_progress(kwargs, git, progress): +def add_progress(kwargs: Any, git: Git, progress: RemoteProgress) -> Any: """Add the --progress flag to the given kwargs dict if supported by the git command. If the actual progress in the given progress instance is not given, we do not request any progress @@ -63,7 +71,7 @@ def add_progress(kwargs, git, progress): #} END utilities -def to_progress_instance(progress): +def to_progress_instance(progress: Optional[RemoteProgress]) -> Union[RemoteProgress, CallableRemoteProgress]: """Given the 'progress' return a suitable object derived from RemoteProgress(). """ @@ -224,7 +232,7 @@ class FetchInfo(object): } @classmethod - def refresh(cls): + def refresh(cls) -> bool: """This gets called by the refresh function (see the top level __init__). """ @@ -247,7 +255,8 @@ def refresh(cls): return True - def __init__(self, ref, flags, note='', old_commit=None, remote_ref_path=None): + def __init__(self, ref: SymbolicReference, flags: Set[int], note: str = '', old_commit: Optional[Commit] = None, + remote_ref_path: Optional[PathLike] = None): """ Initialize a new instance """ @@ -257,16 +266,16 @@ def __init__(self, ref, flags, note='', old_commit=None, remote_ref_path=None): self.old_commit = old_commit self.remote_ref_path = remote_ref_path - def __str__(self): + def __str__(self) -> str: return self.name @property - def name(self): + def name(self) -> str: """:return: Name of our remote ref""" return self.ref.name @property - def commit(self): + def commit(self) -> 'Commit': """:return: Commit of our remote ref""" return self.ref.commit @@ -409,16 +418,6 @@ def __init__(self, repo, name): self.repo = repo # type: 'Repo' self.name = name - if is_win: - # some oddity: on windows, python 2.5, it for some reason does not realize - # that it has the config_writer property, but instead calls __getattr__ - # which will not yield the expected results. 'pinging' the members - # with a dir call creates the config_writer property that we require - # ... bugs like these make me wonder whether python really wants to be used - # for production. It doesn't happen on linux though. - dir(self) - # END windows special handling - def __getattr__(self, attr): """Allows to call this instance like remote.special( \\*args, \\*\\*kwargs) to call git-remote special self.name""" diff --git a/git/repo/base.py b/git/repo/base.py index 253631063..ed0a810e4 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -4,11 +4,6 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - -from git.objects.tag import TagObject -from git.objects.blob import Blob -from git.objects.tree import Tree -from git.refs.symbolic import SymbolicReference import logging import os import re @@ -30,38 +25,44 @@ from git.objects import Submodule, RootModule, Commit from git.refs import HEAD, Head, Reference, TagReference from git.remote import Remote, add_progress, to_progress_instance -from git.util import Actor, IterableList, finalize_process, decygpath, hex_to_bin, expand_path +from git.util import Actor, finalize_process, decygpath, hex_to_bin, expand_path, remove_password_if_present import os.path as osp from .fun import rev_parse, is_git_dir, find_submodule_git_dir, touch, find_worktree_git_dir import gc import gitdb -# Typing ------------------------------------------------------------------- - -from typing import (Any, BinaryIO, Callable, Dict, Iterator, List, Mapping, Optional, - TextIO, Tuple, Type, Union, NamedTuple, cast,) -from typing_extensions import Literal -from git.types import PathLike, TBD - -Lit_config_levels = Literal['system', 'global', 'user', 'repository'] - +# typing ------------------------------------------------------ -# -------------------------------------------------------------------------- +from git.compat.typing import Literal +from git.types import TBD, PathLike +from typing import (Any, BinaryIO, Callable, Dict, + Iterator, List, Mapping, Optional, + TextIO, Tuple, Type, Union, + NamedTuple, cast, TYPE_CHECKING) +if TYPE_CHECKING: # only needed for types + from git.util import IterableList + from git.refs.symbolic import SymbolicReference + from git.objects import TagObject, Blob, Tree # NOQA: F401 -class BlameEntry(NamedTuple): - commit: Dict[str, TBD] # Any == 'Commit' type? - linenos: range - orig_path: Optional[str] - orig_linenos: range +Lit_config_levels = Literal['system', 'global', 'user', 'repository'] +# ----------------------------------------------------------- log = logging.getLogger(__name__) __all__ = ('Repo',) +BlameEntry = NamedTuple('BlameEntry', [ + ('commit', Dict[str, TBD]), + ('linenos', range), + ('orig_path', Optional[str]), + ('orig_linenos', range)] +) + + class Repo(object): """Represents a git repository and allows you to query references, gather commit information, generate diffs, create and clone repositories query @@ -221,10 +222,11 @@ def __init__(self, path: Optional[PathLike] = None, odbt: Type[GitCmdObjectDB] = self.git = self.GitCommandWrapperType(self.working_dir) # special handling, in special times - args = [osp.join(self.common_dir, 'objects')] # type: List[Union[str, Git]] + rootpath = osp.join(self.common_dir, 'objects') if issubclass(odbt, GitCmdObjectDB): - args.append(self.git) - self.odb = odbt(*args) + self.odb = odbt(rootpath, self.git) + else: + self.odb = odbt(rootpath) def __enter__(self) -> 'Repo': return self @@ -266,13 +268,14 @@ def __hash__(self) -> int: # Description property def _get_description(self) -> str: - filename = osp.join(self.git_dir, 'description') if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, 'description') with open(filename, 'rb') as fp: return fp.read().rstrip().decode(defenc) def _set_description(self, descr: str) -> None: - - filename = osp.join(self.git_dir, 'description') if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, 'description') with open(filename, 'wb') as fp: fp.write((descr + '\n').encode(defenc)) @@ -306,7 +309,7 @@ def bare(self) -> bool: return self._bare @property - def heads(self) -> IterableList: + def heads(self) -> 'IterableList': """A list of ``Head`` objects representing the branch heads in this repo @@ -314,7 +317,7 @@ def heads(self) -> IterableList: return Head.list_items(self) @property - def references(self) -> IterableList: + def references(self) -> 'IterableList': """A list of Reference objects representing tags, heads and remote references. :return: IterableList(Reference, ...)""" @@ -327,19 +330,19 @@ def references(self) -> IterableList: branches = heads @property - def index(self) -> IndexFile: + def index(self) -> 'IndexFile': """:return: IndexFile representing this repository's index. :note: This property can be expensive, as the returned ``IndexFile`` will be reinitialized. It's recommended to re-use the object.""" return IndexFile(self) @property - def head(self) -> HEAD: + def head(self) -> 'HEAD': """:return: HEAD Object pointing to the current head reference""" return HEAD(self, 'HEAD') @property - def remotes(self) -> IterableList: + def remotes(self) -> 'IterableList': """A list of Remote objects allowing to access and manipulate remotes :return: ``git.IterableList(Remote, ...)``""" return Remote.list_items(self) @@ -355,13 +358,13 @@ def remote(self, name: str = 'origin') -> 'Remote': #{ Submodules @property - def submodules(self) -> IterableList: + def submodules(self) -> 'IterableList': """ :return: git.IterableList(Submodule, ...) of direct submodules available from the current head""" return Submodule.list_items(self) - def submodule(self, name: str) -> IterableList: + def submodule(self, name: str) -> 'IterableList': """ :return: Submodule with the given name :raise ValueError: If no such submodule exists""" try: @@ -393,7 +396,7 @@ def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator: #}END submodules @property - def tags(self) -> IterableList: + def tags(self) -> 'IterableList': """A list of ``Tag`` objects that are available in this repo :return: ``git.IterableList(TagReference, ...)`` """ return TagReference.list_items(self) @@ -405,14 +408,14 @@ def tag(self, path: PathLike) -> TagReference: def create_head(self, path: PathLike, commit: str = 'HEAD', force: bool = False, logmsg: Optional[str] = None - ) -> SymbolicReference: + ) -> 'SymbolicReference': """Create a new head within the repository. For more documentation, please see the Head.create method. :return: newly created Head Reference""" return Head.create(self, path, commit, force, logmsg) - def delete_head(self, *heads: HEAD, **kwargs: Any) -> None: + def delete_head(self, *heads: 'SymbolicReference', **kwargs: Any) -> None: """Delete the given heads :param kwargs: Additional keyword arguments to be passed to git-branch""" @@ -458,12 +461,11 @@ def _get_config_path(self, config_level: Lit_config_levels) -> str: elif config_level == "global": return osp.normpath(osp.expanduser("~/.gitconfig")) elif config_level == "repository": - if self._common_dir: - return osp.normpath(osp.join(self._common_dir, "config")) - elif self.git_dir: - return osp.normpath(osp.join(self.git_dir, "config")) - else: + repo_dir = self._common_dir or self.git_dir + if not repo_dir: raise NotADirectoryError + else: + return osp.normpath(osp.join(repo_dir, "config")) raise ValueError("Invalid configuration level: %r" % config_level) @@ -503,7 +505,8 @@ def config_writer(self, config_level: Lit_config_levels = "repository") -> GitCo repository = configuration file for this repository only""" return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self) - def commit(self, rev: Optional[TBD] = None,) -> Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]: + def commit(self, rev: Optional[TBD] = None + ) -> Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree']: """The Commit object for the specified revision :param rev: revision specifier, see git-rev-parse for viable options. @@ -536,7 +539,7 @@ def tree(self, rev: Union['Commit', 'Tree', None] = None) -> 'Tree': return self.rev_parse(str(rev) + "^{tree}") def iter_commits(self, rev: Optional[TBD] = None, paths: Union[PathLike, List[PathLike]] = '', - **kwargs: Any,) -> Iterator[Commit]: + **kwargs: Any) -> Iterator[Commit]: """A list of Commit objects representing the history of a given ref/commit :param rev: @@ -560,8 +563,8 @@ def iter_commits(self, rev: Optional[TBD] = None, paths: Union[PathLike, List[Pa return Commit.iter_items(self, rev, paths, **kwargs) - def merge_base(self, *rev: TBD, **kwargs: Any, - ) -> List[Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]]: + def merge_base(self, *rev: TBD, **kwargs: Any + ) -> List[Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree', None]]: """Find the closest common ancestor for the given revision (e.g. Commits, Tags, References, etc) :param rev: At least two revs to find the common ancestor for. @@ -574,7 +577,7 @@ def merge_base(self, *rev: TBD, **kwargs: Any, raise ValueError("Please specify at least two revs, got only %i" % len(rev)) # end handle input - res = [] # type: List[Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]] + res = [] # type: List[Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree', None]] try: lines = self.git.merge_base(*rev, **kwargs).splitlines() # List[str] except GitCommandError as err: @@ -608,11 +611,13 @@ def is_ancestor(self, ancestor_rev: 'Commit', rev: 'Commit') -> bool: return True def _get_daemon_export(self) -> bool: - filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) return osp.exists(filename) def _set_daemon_export(self, value: object) -> None: - filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) if self.git_dir else "" + if self.git_dir: + filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) fileexists = osp.exists(filename) if value and not fileexists: touch(filename) @@ -628,7 +633,8 @@ def _get_alternates(self) -> List[str]: """The list of alternates for this repo from which objects can be retrieved :return: list of strings being pathnames of alternates""" - alternates_path = osp.join(self.git_dir, 'objects', 'info', 'alternates') if self.git_dir else "" + if self.git_dir: + alternates_path = osp.join(self.git_dir, 'objects', 'info', 'alternates') if osp.exists(alternates_path): with open(alternates_path, 'rb') as f: @@ -768,7 +774,7 @@ def blame_incremental(self, rev: TBD, file: TBD, **kwargs: Any) -> Optional[Iter should get a continuous range spanning all line numbers in the file. """ data = self.git.blame(rev, '--', file, p=True, incremental=True, stdout_as_string=False, **kwargs) - commits = {} + commits = {} # type: Dict[str, TBD] stream = (line for line in data.split(b'\n') if line) while True: @@ -776,10 +782,11 @@ def blame_incremental(self, rev: TBD, file: TBD, **kwargs: Any) -> Optional[Iter line = next(stream) # when exhausted, causes a StopIteration, terminating this function except StopIteration: return - hexsha, orig_lineno, lineno, num_lines = line.split() - lineno = int(lineno) - num_lines = int(num_lines) - orig_lineno = int(orig_lineno) + split_line = line.split() # type: Tuple[str, str, str, str] + hexsha, orig_lineno_str, lineno_str, num_lines_str = split_line + lineno = int(lineno_str) + num_lines = int(num_lines_str) + orig_lineno = int(orig_lineno_str) if hexsha not in commits: # Now read the next few lines and build up a dict of properties # for this commit @@ -871,12 +878,10 @@ def blame(self, rev: TBD, file: TBD, incremental: bool = False, **kwargs: Any digits = parts[-1].split(" ") if len(digits) == 3: info = {'id': firstpart} - blames.append([None, [""]]) - elif not info or info['id'] != firstpart: + blames.append([None, []]) + elif info['id'] != firstpart: info = {'id': firstpart} - commits_firstpart = commits.get(firstpart) - blames.append([commits_firstpart, []]) - + blames.append([commits.get(firstpart), []]) # END blame data initialization else: m = self.re_author_committer_start.search(firstpart) @@ -933,8 +938,6 @@ def blame(self, rev: TBD, file: TBD, incremental: bool = False, **kwargs: Any blames[-1][0] = c if blames[-1][1] is not None: blames[-1][1].append(line) - else: - blames[-1][1] = [line] info = {'id': sha} # END if we collected commit info # END distinguish filename,summary,rest @@ -944,7 +947,7 @@ def blame(self, rev: TBD, file: TBD, incremental: bool = False, **kwargs: Any @classmethod def init(cls, path: PathLike = None, mkdir: bool = True, odbt: Type[GitCmdObjectDB] = GitCmdObjectDB, - expand_vars: bool = True, **kwargs: Any,) -> 'Repo': + expand_vars: bool = True, **kwargs: Any) -> 'Repo': """Initialize a git repository at the given path if specified :param path: @@ -983,12 +986,8 @@ def init(cls, path: PathLike = None, mkdir: bool = True, odbt: Type[GitCmdObject @classmethod def _clone(cls, git: 'Git', url: PathLike, path: PathLike, odb_default_type: Type[GitCmdObjectDB], - progress: Optional[Callable], - multi_options: Optional[List[str]] = None, **kwargs: Any, + progress: Optional[Callable], multi_options: Optional[List[str]] = None, **kwargs: Any ) -> 'Repo': - if progress is not None: - progress_checked = to_progress_instance(progress) - odbt = kwargs.pop('odbt', odb_default_type) # when pathlib.Path or other classbased path is passed @@ -1011,13 +1010,16 @@ def _clone(cls, git: 'Git', url: PathLike, path: PathLike, odb_default_type: Typ if multi_options: multi = ' '.join(multi_options).split(' ') proc = git.clone(multi, Git.polish_url(url), clone_path, with_extended_output=True, as_process=True, - v=True, universal_newlines=True, **add_progress(kwargs, git, progress_checked)) - if progress_checked: - handle_process_output(proc, None, progress_checked.new_message_handler(), + v=True, universal_newlines=True, **add_progress(kwargs, git, progress)) + if progress: + handle_process_output(proc, None, to_progress_instance(progress).new_message_handler(), finalize_process, decode_streams=False) else: (stdout, stderr) = proc.communicate() - log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''), stdout) + cmdline = getattr(proc, 'args', '') + cmdline = remove_password_if_present(cmdline) + + log.debug("Cmd(%s)'s unused stdout: %s", cmdline, stdout) finalize_process(proc, stderr=stderr) # our git command could have a different working dir than our actual @@ -1130,13 +1132,14 @@ def __repr__(self) -> str: clazz = self.__class__ return '<%s.%s %r>' % (clazz.__module__, clazz.__name__, self.git_dir) - def currently_rebasing_on(self) -> Union[SymbolicReference, Commit, TagObject, Blob, Tree, None]: + def currently_rebasing_on(self) -> Union['SymbolicReference', Commit, 'TagObject', 'Blob', 'Tree', None]: """ :return: The commit which is currently being replayed while rebasing. None if we are not currently rebasing. """ - rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") if self.git_dir else "" + if self.git_dir: + rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") if not osp.isfile(rebase_head_file): return None return self.commit(open(rebase_head_file, "rt").readline().strip()) diff --git a/git/repo/fun.py b/git/repo/fun.py index b81845932..703940819 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -18,11 +18,12 @@ # Typing ---------------------------------------------------------------------- -from .base import Repo -from git.db import GitCmdObjectDB -from git.objects import Commit, TagObject, Blob, Tree -from typing import AnyStr, Union, Optional, cast +from typing import AnyStr, Union, Optional, cast, TYPE_CHECKING from git.types import PathLike +if TYPE_CHECKING: + from .base import Repo + from git.db import GitCmdObjectDB + from git.objects import Commit, TagObject, Blob, Tree # ---------------------------------------------------------------------------- @@ -102,7 +103,7 @@ def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]: return None -def short_to_long(odb: GitCmdObjectDB, hexsha: AnyStr) -> Optional[bytes]: +def short_to_long(odb: 'GitCmdObjectDB', hexsha: AnyStr) -> Optional[bytes]: """:return: long hexadecimal sha1 from the given less-than-40 byte hexsha or None if no candidate could be found. :param hexsha: hexsha with less than 40 byte""" @@ -113,8 +114,8 @@ def short_to_long(odb: GitCmdObjectDB, hexsha: AnyStr) -> Optional[bytes]: # END exception handling -def name_to_object(repo: Repo, name: str, return_ref: bool = False, - ) -> Union[SymbolicReference, Commit, TagObject, Blob, Tree]: +def name_to_object(repo: 'Repo', name: str, return_ref: bool = False + ) -> Union[SymbolicReference, 'Commit', 'TagObject', 'Blob', 'Tree']: """ :return: object specified by the given name, hexshas ( short and long ) as well as references are supported @@ -161,7 +162,7 @@ def name_to_object(repo: Repo, name: str, return_ref: bool = False, return Object.new_from_sha(repo, hex_to_bin(hexsha)) -def deref_tag(tag: Tag) -> TagObject: +def deref_tag(tag: Tag) -> 'TagObject': """Recursively dereference a tag and return the resulting object""" while True: try: @@ -172,7 +173,7 @@ def deref_tag(tag: Tag) -> TagObject: return tag -def to_commit(obj: Object) -> Union[Commit, TagObject]: +def to_commit(obj: Object) -> Union['Commit', 'TagObject']: """Convert the given object to a commit if possible and return it""" if obj.type == 'tag': obj = deref_tag(obj) @@ -183,7 +184,7 @@ def to_commit(obj: Object) -> Union[Commit, TagObject]: return obj -def rev_parse(repo: Repo, rev: str) -> Union[Commit, Tag, Tree, Blob]: +def rev_parse(repo: 'Repo', rev: str) -> Union['Commit', 'Tag', 'Tree', 'Blob']: """ :return: Object at the given revision, either Commit, Tag, Tree or Blob :param rev: git-rev-parse compatible revision specification as string, please see diff --git a/git/types.py b/git/types.py index dc44c1231..3e33ae0c9 100644 --- a/git/types.py +++ b/git/types.py @@ -1,6 +1,20 @@ -import os # @UnusedImport ## not really unused, is in type string +# -*- coding: utf-8 -*- +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php + +import os +import sys from typing import Union, Any TBD = Any -PathLike = Union[str, 'os.PathLike[str]'] + +if sys.version_info[:2] < (3, 6): + # os.PathLike (PEP-519) only got introduced with Python 3.6 + PathLike = str +elif sys.version_info[:2] < (3, 9): + # Python >= 3.6, < 3.9 + PathLike = Union[str, os.PathLike] +elif sys.version_info[:2] >= (3, 9): + # os.PathLike only becomes subscriptable from Python 3.9 onwards + PathLike = Union[str, os.PathLike[str]] diff --git a/git/util.py b/git/util.py index 2b0c81715..af4990286 100644 --- a/git/util.py +++ b/git/util.py @@ -3,7 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -from git.remote import Remote + import contextlib from functools import wraps import getpass @@ -17,11 +17,15 @@ from sys import maxsize import time from unittest import SkipTest +from urllib.parse import urlsplit, urlunsplit # typing --------------------------------------------------------- + from typing import (Any, AnyStr, BinaryIO, Callable, Dict, Generator, IO, List, - NoReturn, Optional, Pattern, Sequence, Tuple, Union, cast) -from git.repo.base import Repo + NoReturn, Optional, Pattern, Sequence, Tuple, Union, cast, TYPE_CHECKING) +if TYPE_CHECKING: + from git.remote import Remote + from git.repo.base import Repo from .types import PathLike, TBD # --------------------------------------------------------------------- @@ -74,7 +78,7 @@ def unbare_repo(func: Callable) -> Callable: encounter a bare repository""" @wraps(func) - def wrapper(self: Remote, *args: Any, **kwargs: Any) -> TBD: + def wrapper(self: 'Remote', *args: Any, **kwargs: Any) -> TBD: if self.repo.bare: raise InvalidGitRepositoryError("Method '%s' cannot operate on bare repositories" % func.__name__) # END bare method @@ -359,6 +363,34 @@ def expand_path(p: PathLike, expand_vars: bool = True) -> Optional[PathLike]: except Exception: return None + +def remove_password_if_present(cmdline): + """ + Parse any command line argument and if on of the element is an URL with a + password, replace it by stars (in-place). + + If nothing found just returns the command line as-is. + + This should be used for every log line that print a command line. + """ + new_cmdline = [] + for index, to_parse in enumerate(cmdline): + new_cmdline.append(to_parse) + try: + url = urlsplit(to_parse) + # Remove password from the URL if present + if url.password is None: + continue + + edited_url = url._replace( + netloc=url.netloc.replace(url.password, "*****")) + new_cmdline[index] = urlunsplit(edited_url) + except ValueError: + # This is not a valid URL + continue + return new_cmdline + + #} END utilities #{ Classes @@ -686,7 +718,7 @@ def __init__(self, total: Dict[str, Dict[str, int]], files: Dict[str, Dict[str, self.files = files @classmethod - def _list_from_string(cls, repo: Repo, text: str) -> 'Stats': + def _list_from_string(cls, repo: 'Repo', text: str) -> 'Stats': """Create a Stat object from output retrieved by git-diff. :return: git.Stat""" @@ -924,6 +956,7 @@ def __getitem__(self, index: Union[int, slice, str]) -> Any: def __delitem__(self, index: Union[int, str, slice]) -> None: + delindex = cast(int, index) if not isinstance(index, int): delindex = -1 assert not isinstance(index, slice) @@ -949,7 +982,7 @@ class Iterable(object): _id_attribute_ = "attribute that most suitably identifies your instance" @classmethod - def list_items(cls, repo: Repo, *args: Any, **kwargs: Any) -> 'IterableList': + def list_items(cls, repo: 'Repo', *args: Any, **kwargs: Any) -> 'IterableList': """ Find all items of this type - subclasses can specify args and kwargs differently. If no args are given, subclasses are obliged to return all items if no additional @@ -963,7 +996,7 @@ def list_items(cls, repo: Repo, *args: Any, **kwargs: Any) -> 'IterableList': return out_list @classmethod - def iter_items(cls, repo: Repo, *args: Any, **kwargs: Any) -> NoReturn: + def iter_items(cls, repo: 'Repo', *args: Any, **kwargs: Any) -> NoReturn: """For more information about the arguments, see list_items :return: iterator yielding Items""" raise NotImplementedError("To be implemented by Subclass") diff --git a/mypy.ini b/mypy.ini index 47c0fb0c0..b63d68fd3 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,9 @@ [mypy] -disallow_untyped_defs = True +# TODO: enable when we've fully annotated everything +#disallow_untyped_defs = True -mypy_path = 'git' +# TODO: remove when 'gitdb' is fully annotated +[mypy-gitdb.*] +ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt index c4e8340d8..d980f6682 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ gitdb>=4.0.1,<5 +typing-extensions>=3.7.4.0;python_version<"3.8" diff --git a/test-requirements.txt b/test-requirements.txt index abda95cf0..e06d2be14 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,3 +4,5 @@ flake8 tox virtualenv nose +gitdb>=4.0.1,<5 +typing-extensions>=3.7.4.0;python_version<"3.8" diff --git a/test/fixtures/diff_file_with_colon b/test/fixtures/diff_file_with_colon new file mode 100644 index 0000000000000000000000000000000000000000..4058b1715bd164ef8c13c73a458426bc43dcc5d0 GIT binary patch literal 351 zcmY+9L2g4a2nBN#pCG{o^G)v1GgM%p{ZiCa>X(|{zOIx_Suo3abFBbORGtVHk0xf# zt1}_luqGPY*44*sL1T85T9COlPLmCE!c-N<>|J8`qgK z10mULiQo3)5|87u=(dFaN+(aTwH5}_u&!;nb*vEUE4_8K%$Smeu+|L?+-VFY9wIF} c#^^1?Cph;RUU3PJ_&P3s@74Fr^XJd$7YhDgzW@LL literal 0 HcmV?d00001 diff --git a/test/test_clone.py b/test/test_clone.py new file mode 100644 index 000000000..e9f6714d3 --- /dev/null +++ b/test/test_clone.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php + +from pathlib import Path +import re + +import git + +from .lib import ( + TestBase, + with_rw_directory, +) + + +class TestClone(TestBase): + @with_rw_directory + def test_checkout_in_non_empty_dir(self, rw_dir): + non_empty_dir = Path(rw_dir) + garbage_file = non_empty_dir / 'not-empty' + garbage_file.write_text('Garbage!') + + # Verify that cloning into the non-empty dir fails while complaining about + # the target directory not being empty/non-existent + try: + self.rorepo.clone(non_empty_dir) + except git.GitCommandError as exc: + self.assertTrue(exc.stderr, "GitCommandError's 'stderr' is unexpectedly empty") + expr = re.compile(r'(?is).*\bfatal:\s+destination\s+path\b.*\bexists\b.*\bnot\b.*\bempty\s+directory\b') + self.assertTrue(expr.search(exc.stderr), '"%s" does not match "%s"' % (expr.pattern, exc.stderr)) + else: + self.fail("GitCommandError not raised") diff --git a/test/test_diff.py b/test/test_diff.py index c6c9b67a0..9b20893a4 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -7,6 +7,7 @@ import ddt import shutil import tempfile +import unittest from git import ( Repo, GitCommandError, @@ -220,6 +221,12 @@ def test_diff_index_raw_format(self): self.assertIsNotNone(res[0].deleted_file) self.assertIsNone(res[0].b_path,) + @unittest.skip("This currently fails and would need someone to improve diff parsing") + def test_diff_file_with_colon(self): + output = fixture('diff_file_with_colon') + res = [] + Diff._handle_diff_line(output, None, res) + def test_diff_initial_commit(self): initial_commit = self.rorepo.commit('33ebe7acec14b25c5f84f35a664803fcab2f7781') diff --git a/test/test_repo.py b/test/test_repo.py index d5ea8664a..8dc178337 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -238,6 +238,21 @@ def test_clone_from_with_path_contains_unicode(self): except UnicodeEncodeError: self.fail('Raised UnicodeEncodeError') + @with_rw_directory + def test_leaking_password_in_clone_logs(self, rw_dir): + password = "fakepassword1234" + try: + Repo.clone_from( + url="https://fakeuser:{}@fakerepo.example.com/testrepo".format( + password), + to_path=rw_dir) + except GitCommandError as err: + assert password not in str(err), "The error message '%s' should not contain the password" % err + # Working example from a blank private project + Repo.clone_from( + url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", + to_path=rw_dir) + @with_rw_repo('HEAD') def test_max_chunk_size(self, repo): class TestOutputStream(TestBase): diff --git a/test/test_util.py b/test/test_util.py index 5eba6c500..ddc5f628f 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -30,7 +30,8 @@ Actor, IterableList, cygpath, - decygpath + decygpath, + remove_password_if_present, ) @@ -322,3 +323,20 @@ def test_pickle_tzoffset(self): t2 = pickle.loads(pickle.dumps(t1)) self.assertEqual(t1._offset, t2._offset) self.assertEqual(t1._name, t2._name) + + def test_remove_password_from_command_line(self): + password = "fakepassword1234" + url_with_pass = "https://fakeuser:{}@fakerepo.example.com/testrepo".format(password) + url_without_pass = "https://fakerepo.example.com/testrepo" + + cmd_1 = ["git", "clone", "-v", url_with_pass] + cmd_2 = ["git", "clone", "-v", url_without_pass] + cmd_3 = ["no", "url", "in", "this", "one"] + + redacted_cmd_1 = remove_password_if_present(cmd_1) + assert password not in " ".join(redacted_cmd_1) + # Check that we use a copy + assert cmd_1 is not redacted_cmd_1 + assert password in " ".join(cmd_1) + assert cmd_2 == remove_password_if_present(cmd_2) + assert cmd_3 == remove_password_if_present(cmd_3) diff --git a/tox.ini b/tox.ini index ad126ed4e..a0cb1c9f1 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,14 @@ commands = coverage run --omit="git/test/*" -m unittest --buffer {posargs} [testenv:flake8] commands = flake8 --ignore=W293,E265,E266,W503,W504,E731 {posargs} +[testenv:type] +description = type check ourselves +deps = + {[testenv]deps} + mypy +commands = + mypy -p git + [testenv:venv] commands = {posargs} @@ -23,6 +31,7 @@ commands = {posargs} # E266 = too many leading '#' for block comment # E731 = do not assign a lambda expression, use a def # W293 = Blank line contains whitespace -ignore = E265,W293,E266,E731 +# W504 = Line break after operator +ignore = E265,W293,E266,E731, W504 max-line-length = 120 exclude = .tox,.venv,build,dist,doc,git/ext/