From 5077fc7e4031e53f730676df4d8df5165b1d36cc Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Sun, 29 May 2016 13:34:35 +0100 Subject: [PATCH 1/9] Return all the stderr messge after an error is detected for pull() --- git/cmd.py | 6 +++--- git/remote.py | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index c29e34854..821bf2992 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -307,7 +307,7 @@ def __del__(self): def __getattr__(self, attr): return getattr(self.proc, attr) - def wait(self, stderr=None): + def wait(self, stderr=''): """Wait for the process and return its status code. :param stderr: Previously read value of stderr, in case stderr is already closed. @@ -317,7 +317,7 @@ def wait(self, stderr=None): def read_all_from_possibly_closed_stream(stream): try: - return stream.read() + return stderr + stream.read() except ValueError: return stderr or '' @@ -678,7 +678,7 @@ def _kill_process(pid): # strip trailing "\n" if stderr_value.endswith(b"\n"): stderr_value = stderr_value[:-1] - status = proc.wait() + status = proc.wait(stderr=stderr_value) # END stdout handling finally: proc.stdout.close() diff --git a/git/remote.py b/git/remote.py index 30e32ae3d..f23f50a24 100644 --- a/git/remote.py +++ b/git/remote.py @@ -570,11 +570,16 @@ def _get_fetch_info_from_stderr(self, proc, progress): progress_handler = progress.new_message_handler() + error_message = None + stderr_text = None + for line in proc.stderr: line = force_text(line) for pline in progress_handler(line): if line.startswith('fatal:') or line.startswith('error:'): - raise GitCommandError(("Error when fetching: %s" % line,), 2) + error_message = "Error when fetching: %s" % (line,) + break + # END handle special messages for cmd in cmds: if len(line) > 1 and line[0] == ' ' and line[1] == cmd: @@ -582,9 +587,19 @@ def _get_fetch_info_from_stderr(self, proc, progress): continue # end find command code # end for each comand code we know + + if error_message is not None: + break # end for each line progress didn't handle + + if error_message is not None: + stderr_text = proc.stderr.read() + # end - finalize_process(proc) + finalize_process(proc, stderr=stderr_text) + + if error_message is not None: + raise GitCommandError( error_message, 2, stderr=stderr_text ) # read head information fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') From 78f3f38d18fc88fd639af8a6c1ef757d2ffe51d6 Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Sun, 29 May 2016 13:59:53 +0100 Subject: [PATCH 2/9] Return stderr lines from a pull() call that fails --- git/remote.py | 4 ++++ git/util.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index f23f50a24..1ef62409b 100644 --- a/git/remote.py +++ b/git/remote.py @@ -646,6 +646,10 @@ def stdout_handler(line): try: handle_process_output(proc, stdout_handler, progress_handler, finalize_process) + except GitCommandError as err: + # convert any error from wait() into the same error with stdout lines + raise GitCommandError( err.command, err.status, progress.get_stderr() ) + except Exception: if len(output) == 0: raise diff --git a/git/util.py b/git/util.py index 5ed014fc2..f185156c1 100644 --- a/git/util.py +++ b/git/util.py @@ -173,13 +173,17 @@ class RemoteProgress(object): DONE_TOKEN = 'done.' TOKEN_SEPARATOR = ', ' - __slots__ = ("_cur_line", "_seen_ops") + __slots__ = ("_cur_line", "_seen_ops", "_error_lines") re_op_absolute = re.compile(r"(remote: )?([\w\s]+):\s+()(\d+)()(.*)") re_op_relative = re.compile(r"(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)") def __init__(self): self._seen_ops = list() self._cur_line = None + self._error_lines = [] + + def get_stderr(self): + return '\n'.join(self._error_lines) def _parse_progress_line(self, line): """Parse progress information from the given line as retrieved by git-push @@ -190,6 +194,10 @@ def _parse_progress_line(self, line): # Counting objects: 4, done. # Compressing objects: 50% (1/2) \rCompressing objects: 100% (2/2) \rCompressing objects: 100% (2/2), done. self._cur_line = line + if len(self._error_lines) > 0 or self._cur_line.startswith( ('error:', 'fatal:') ): + self._error_lines.append( self._cur_line ) + return [] + sub_lines = line.split('\r') failed_lines = list() for sline in sub_lines: From 46201b346fec29f9cb740728a3c20266094d58b2 Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Mon, 30 May 2016 15:05:32 +0100 Subject: [PATCH 3/9] Fix flake8 complaints --- git/remote.py | 4 ++-- git/util.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/git/remote.py b/git/remote.py index 1ef62409b..ba51fe925 100644 --- a/git/remote.py +++ b/git/remote.py @@ -599,7 +599,7 @@ def _get_fetch_info_from_stderr(self, proc, progress): finalize_process(proc, stderr=stderr_text) if error_message is not None: - raise GitCommandError( error_message, 2, stderr=stderr_text ) + raise GitCommandError(error_message, 2, stderr=stderr_text) # read head information fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') @@ -648,7 +648,7 @@ def stdout_handler(line): handle_process_output(proc, stdout_handler, progress_handler, finalize_process) except GitCommandError as err: # convert any error from wait() into the same error with stdout lines - raise GitCommandError( err.command, err.status, progress.get_stderr() ) + raise GitCommandError(err.command, err.status, progress.get_stderr()) except Exception: if len(output) == 0: diff --git a/git/util.py b/git/util.py index f185156c1..2f8945765 100644 --- a/git/util.py +++ b/git/util.py @@ -194,8 +194,8 @@ def _parse_progress_line(self, line): # Counting objects: 4, done. # Compressing objects: 50% (1/2) \rCompressing objects: 100% (2/2) \rCompressing objects: 100% (2/2), done. self._cur_line = line - if len(self._error_lines) > 0 or self._cur_line.startswith( ('error:', 'fatal:') ): - self._error_lines.append( self._cur_line ) + if len(self._error_lines) > 0 or self._cur_line.startswith(('error:', 'fatal:')): + self._error_lines.append(self._cur_line) return [] sub_lines = line.split('\r') From 08a0fad2c9dcdfe0bbc980b8cd260b4be5582381 Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Mon, 30 May 2016 15:49:40 +0100 Subject: [PATCH 4/9] Make sure that stderr is converted to bytes remove stderr for a wait() that is not the GitPython wrapper. --- git/cmd.py | 15 ++++++++++++--- git/util.py | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 821bf2992..e3b39bda4 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -307,19 +307,28 @@ def __del__(self): def __getattr__(self, attr): return getattr(self.proc, attr) - def wait(self, stderr=''): + def wait(self, stderr=b''): """Wait for the process and return its status code. :param stderr: Previously read value of stderr, in case stderr is already closed. :warn: may deadlock if output or error pipes are used and not handled separately. :raise GitCommandError: if the return status is not 0""" + + # stderr must be a bytes object as it will + # combined with more data from the process and + # decoded by the caller + if stderr is None: + stderr = b'' + elif type(stderr) == unicode: + stderr = stderr.encode(defenc) + status = self.proc.wait() def read_all_from_possibly_closed_stream(stream): try: return stderr + stream.read() except ValueError: - return stderr or '' + return stderr or b'' if status != 0: errstr = read_all_from_possibly_closed_stream(self.proc.stderr) @@ -678,7 +687,7 @@ def _kill_process(pid): # strip trailing "\n" if stderr_value.endswith(b"\n"): stderr_value = stderr_value[:-1] - status = proc.wait(stderr=stderr_value) + status = proc.wait() # END stdout handling finally: proc.stdout.close() diff --git a/git/util.py b/git/util.py index 2f8945765..706518b8e 100644 --- a/git/util.py +++ b/git/util.py @@ -772,10 +772,10 @@ def done(self): self.cv.notify_all() self.cv.release() - def wait(self): + def wait(self, stderr=b''): self.cv.acquire() while self.count > 0: - self.cv.wait() + self.cv.wait(strerr=stderr) self.cv.release() From 4a5cebaeda2c5062fb6c727f457ee3288f6046ef Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Mon, 6 Jun 2016 10:28:29 +0100 Subject: [PATCH 5/9] log all the output from stdout and stderr for debugging process failures --- git/cmd.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index fb00869cf..f992a399b 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -69,6 +69,10 @@ def _bchr(c): # Documentation ## @{ +def _drop_output_handler(line): + pass + + def handle_process_output(process, stdout_handler, stderr_handler, finalizer): """Registers for notifications to lean that process output is ready to read, and dispatches lines to the respective line handlers. We are able to handle carriage returns in case progress is sent by that @@ -79,6 +83,13 @@ def handle_process_output(process, stdout_handler, stderr_handler, finalizer): :param stdout_handler: f(stdout_line_string), or None :param stderr_hanlder: f(stderr_line_string), or None :param finalizer: f(proc) - wait for proc to finish""" + + log.debug('handle_process_output( process=%r, stdout_handler=%r, stderr_handler=%r, finalizer=%r' + % (process, stdout_handler, stderr_handler, finalizer)) + + if stdout_handler is None: + stdout_handler = _drop_output_handler + fdmap = {process.stdout.fileno(): (stdout_handler, [b'']), process.stderr.fileno(): (stderr_handler, [b''])} @@ -119,6 +130,7 @@ def _dispatch_single_line(line, handler): # end single line helper def _dispatch_lines(fno, handler, buf_list): + log.debug('fno=%d, handler=%r, buf_list=%r' % (fno, handler, buf_list)) lc = 0 for line in _read_lines_from_fno(fno, buf_list): _dispatch_single_line(line, handler) @@ -332,6 +344,7 @@ def read_all_from_possibly_closed_stream(stream): if status != 0: errstr = read_all_from_possibly_closed_stream(self.proc.stderr) + log.debug('AutoInterrupt wait stderr: %r' % (errstr,)) raise GitCommandError(self.args, status, errstr) # END status handling return status @@ -618,7 +631,7 @@ def execute(self, command, bufsize=-1, stdin=istream, stderr=PIPE, - stdout=PIPE if with_stdout else open(os.devnull, 'wb'), + stdout=PIPE, shell=self.USE_SHELL, close_fds=(os.name == 'posix'), # unsupported on windows universal_newlines=universal_newlines, From 6891caf73735ea465c909de8dc13129cc98c47f7 Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Mon, 6 Jun 2016 10:45:16 +0100 Subject: [PATCH 6/9] Can get a str object from stream.read rather then bytes. Convert to the expected bytes. --- git/cmd.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index f992a399b..633aedcbe 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -338,7 +338,10 @@ def wait(self, stderr=b''): def read_all_from_possibly_closed_stream(stream): try: - return stderr + stream.read() + last_stderr = stream.read() + if type(last_stderr) == unicode: + last_stderr = last_stderr.encode(defenc) + return stderr + last_stderr except ValueError: return stderr or b'' From 349c209d5ecf7fb1c5e52ef71de9f35fafb45aec Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Mon, 6 Jun 2016 13:49:50 +0100 Subject: [PATCH 7/9] There is no code that will put FOO in the args of the GitCommandError. git can be expected to exit with 128 as its will fail hard without a working ssh command. --- git/test/test_git.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/git/test/test_git.py b/git/test/test_git.py index 2d6ca8bca..acd96915c 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -205,12 +205,7 @@ def test_environment(self, rw_dir): try: remote.fetch() except GitCommandError as err: - if sys.version_info[0] < 3 and sys.platform == 'darwin': - assert 'ssh-origin' in str(err) - assert err.status == 128 - else: - assert 'FOO' in str(err) - assert err.status == 2 + assert err.status == 128 # end # end # end if select.poll exists From 99b4919ed3170f760745e7ec028f1b63cb79455f Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Mon, 6 Jun 2016 13:51:48 +0100 Subject: [PATCH 8/9] Log the value of GIT_SSH as it is as important the comman line itself. --- git/cmd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/git/cmd.py b/git/cmd.py index 633aedcbe..dd4a9755e 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -628,6 +628,7 @@ def execute(self, command, # end handle try: + log.info('env GIT_SSH=%r' % (env.get('GET_SSH', None),)) proc = Popen(command, env=env, cwd=cwd, From d6df9fd2ca016955248c3321161dfbd39658240e Mon Sep 17 00:00:00 2001 From: Barry Scott Date: Mon, 6 Jun 2016 15:39:21 +0100 Subject: [PATCH 9/9] Fif reflog corruption. Only the first line of commit must be appended to the log. Also fix the ValueError to use %r and (value,) idiom. --- git/refs/log.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/git/refs/log.py b/git/refs/log.py index fed136087..3078355d6 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -114,7 +114,7 @@ def from_line(cls, line): newhexsha = info[41:81] for hexsha in (oldhexsha, newhexsha): if not cls._re_hexsha_only.match(hexsha): - raise ValueError("Invalid hexsha: %s" % hexsha) + raise ValueError("Invalid hexsha: %r" % (hexsha,)) # END if hexsha re doesn't match # END for each hexsha @@ -274,11 +274,12 @@ def append_entry(cls, config_reader, filepath, oldbinsha, newbinsha, message): raise ValueError("Shas need to be given in binary format") # END handle sha type assure_directory_exists(filepath, is_file=True) + first_line = message.split('\n')[0] committer = isinstance(config_reader, Actor) and config_reader or Actor.committer(config_reader) entry = RefLogEntry(( bin_to_hex(oldbinsha).decode('ascii'), bin_to_hex(newbinsha).decode('ascii'), - committer, (int(time.time()), time.altzone), message + committer, (int(time.time()), time.altzone), first_line )) lf = LockFile(filepath)