Merge remote-tracking branch 'origin/master' into document-blocking-writedocument-blocking-write

8 files changed, 155 insertions, 131 deletions

diff --git a/pexpect/__init__.py b/pexpect/__init__.py
index 4b153f4..db5be16 100644
--- a/pexpect/__init__.py
+++ b/pexpect/__init__.py
@@ -77,7 +77,7 @@ __all__ = ['ExceptionPexpect', 'EOF', 'TIMEOUT', 'spawn', 'spawnu', 'run', 'runu
            'which', 'split_command_line', '__version__', '__revision__']
 
 def run(command, timeout=30, withexitstatus=False, events=None,
-        extra_args=None, logfile=None, cwd=None, env=None):
+        extra_args=None, logfile=None, cwd=None, env=None, **kwargs):
 
     '''
     This function runs the given command; waits for it to finish; then
@@ -159,29 +159,16 @@ def run(command, timeout=30, withexitstatus=False, events=None,
     sent to the child. 'extra_args' is not used by directly run(). It provides
     a way to pass data to a callback function through run() through the locals
     dictionary passed to a callback.
-    '''
-    return _run(command, timeout=timeout, withexitstatus=withexitstatus,
-                events=events, extra_args=extra_args, logfile=logfile, cwd=cwd,
-                env=env, _spawn=spawn)
-
-def runu(command, timeout=30, withexitstatus=False, events=None,
-        extra_args=None, logfile=None, cwd=None, env=None, **kwargs):
-    """This offers the same interface as :func:`run`, but using unicode.
 
-    Like :class:`spawnu`, you can pass ``encoding`` and ``errors`` parameters,
-    which will be used for both input and output.
-    """
-    return _run(command, timeout=timeout, withexitstatus=withexitstatus,
-                events=events, extra_args=extra_args, logfile=logfile, cwd=cwd,
-                env=env, _spawn=spawnu, **kwargs)
-
-def _run(command, timeout, withexitstatus, events, extra_args, logfile, cwd,
-         env, _spawn, **kwargs):
+    Like :class:`spawn`, passing *encoding* will make it work with unicode
+    instead of bytes. You can pass *codec_errors* to control how errors in
+    encoding and decoding are handled.
+    '''
     if timeout == -1:
-        child = _spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env,
+        child = spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env,
                         **kwargs)
     else:
-        child = _spawn(command, timeout=timeout, maxread=2000, logfile=logfile,
+        child = spawn(command, timeout=timeout, maxread=2000, logfile=logfile,
                 cwd=cwd, env=env, **kwargs)
     if isinstance(events, list):
         patterns= [x for x,y in events]
@@ -232,4 +219,13 @@ def _run(command, timeout, withexitstatus, events, extra_args, logfile, cwd,
     else:
         return child_result
 
+def runu(command, timeout=30, withexitstatus=False, events=None,
+        extra_args=None, logfile=None, cwd=None, env=None, **kwargs):
+    """Deprecated: pass encoding to run() instead.
+    """
+    kwargs.setdefault('encoding', 'utf-8')
+    return run(command, timeout=timeout, withexitstatus=withexitstatus,
+                events=events, extra_args=extra_args, logfile=logfile, cwd=cwd,
+                env=env, **kwargs)
+
 # vim: set shiftround expandtab tabstop=4 shiftwidth=4 ft=python autoindent :
diff --git a/pexpect/async.py b/pexpect/async.py
index 5e5e9ee..ad75994 100644
--- a/pexpect/async.py
+++ b/pexpect/async.py
@@ -37,7 +37,7 @@ class PatternWaiter(asyncio.Protocol):
     
     def data_received(self, data):
         spawn = self.expecter.spawn
-        s = spawn._coerce_read_string(data)
+        s = spawn._decoder.decode(data)
         spawn._log(s, 'read')
 
         if self.fut.done():
diff --git a/pexpect/fdpexpect.py b/pexpect/fdpexpect.py
index 96ca2e1..ca8cf07 100644
--- a/pexpect/fdpexpect.py
+++ b/pexpect/fdpexpect.py
@@ -32,7 +32,8 @@ class fdspawn(SpawnBase):
     descriptor. For example, you could use it to read through a file looking
     for patterns, or to control a modem or serial device. '''
 
-    def __init__ (self, fd, args=None, timeout=30, maxread=2000, searchwindowsize=None, logfile=None):
+    def __init__ (self, fd, args=None, timeout=30, maxread=2000, searchwindowsize=None,
+                  logfile=None, encoding=None, codec_errors='strict'):
         '''This takes a file descriptor (an int) or an object that support the
         fileno() method (returning an int). All Python file-like objects
         support fileno(). '''
@@ -50,7 +51,8 @@ class fdspawn(SpawnBase):
 
         self.args = None
         self.command = None
-        SpawnBase.__init__(self, timeout, maxread, searchwindowsize, logfile)
+        SpawnBase.__init__(self, timeout, maxread, searchwindowsize, logfile,
+                           encoding=encoding, codec_errors=codec_errors)
         self.child_fd = fd
         self.own_fd = False
         self.closed = False
@@ -84,3 +86,28 @@ class fdspawn(SpawnBase):
 
     def terminate (self, force=False):  # pragma: no cover
         raise ExceptionPexpect('This method is not valid for file descriptors.')
+    
+    # These four methods are left around for backwards compatibility, but not
+    # documented as part of fdpexpect. You're encouraged to use os.write#
+    # directly.
+    def send(self, s):
+        "Write to fd, return number of bytes written"
+        s = self._coerce_send_string(s)
+        self._log(s, 'send')
+        
+        b = self._encoder.encode(s, final=False)
+        return os.write(self.child_fd, b)
+    
+    def sendline(self, s):
+        "Write to fd with trailing newline, return number of bytes written"
+        s = self._coerce_send_string(s)
+        return self.send(s + self.linesep)
+    
+    def write(self, s):
+        "Write to fd, return None"
+        self.send(s)
+    
+    def writelines(self, sequence):
+        "Call self.write() for each item in sequence"
+        for s in sequence:
+            self.write(s)
diff --git a/pexpect/pty_spawn.py b/pexpect/pty_spawn.py
index 0ba1e0c..73dec56 100644
--- a/pexpect/pty_spawn.py
+++ b/pexpect/pty_spawn.py
@@ -14,7 +14,7 @@ import ptyprocess
 from ptyprocess.ptyprocess import use_native_pty_fork
 
 from .exceptions import ExceptionPexpect, EOF, TIMEOUT
-from .spawnbase import SpawnBase, SpawnBaseUnicode
+from .spawnbase import SpawnBase
 from .utils import which, split_command_line
 
 @contextmanager
@@ -30,14 +30,14 @@ PY3 = (sys.version_info[0] >= 3)
 class spawn(SpawnBase):
     '''This is the main class interface for Pexpect. Use this class to start
     and control child applications. '''
-    ptyprocess_class = ptyprocess.PtyProcess
 
     # This is purely informational now - changing it has no effect
     use_native_pty_fork = use_native_pty_fork
 
     def __init__(self, command, args=[], timeout=30, maxread=2000,
                  searchwindowsize=None, logfile=None, cwd=None, env=None,
-                 ignore_sighup=True, echo=True, preexec_fn=None):
+                 ignore_sighup=True, echo=True, preexec_fn=None,
+                 encoding=None, codec_errors='strict'):
         '''This is the constructor. The command parameter may be a string that
         includes a command and any arguments to the command. For example::
 
@@ -117,7 +117,7 @@ class spawn(SpawnBase):
             child = pexpect.spawn('some_command')
             child.logfile_read = sys.stdout
 
-        Remember to use spawnu instead of spawn for the above code if you are
+        You will need to pass an encoding to spawn in the above code if you are
         using Python 3.
 
         To separately log output sent to the child use logfile_send::
@@ -172,7 +172,7 @@ class spawn(SpawnBase):
         signal handlers.
         '''
         super(spawn, self).__init__(timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize,
-                                    logfile=logfile)
+                                    logfile=logfile, encoding=encoding, codec_errors=codec_errors)
         self.STDIN_FILENO = pty.STDIN_FILENO
         self.STDOUT_FILENO = pty.STDOUT_FILENO
         self.STDERR_FILENO = pty.STDERR_FILENO
@@ -277,7 +277,7 @@ class spawn(SpawnBase):
                     preexec_fn()
             kwargs['preexec_fn'] = preexec_wrapper
 
-        self.ptyproc = self.ptyprocess_class.spawn(self.args, env=self.env,
+        self.ptyproc = ptyprocess.PtyProcess.spawn(self.args, env=self.env,
                                                    cwd=self.cwd, **kwargs)
 
         self.pid = self.ptyproc.pid
@@ -503,10 +503,8 @@ class spawn(SpawnBase):
         s = self._coerce_send_string(s)
         self._log(s, 'send')
 
-        return self._send(s)
-
-    def _send(self, s):
-        return os.write(self.child_fd, s)
+        b = self._encoder.encode(s, final=False)
+        return os.write(self.child_fd, b)
 
     def sendline(self, s=''):
         '''Wraps send(), sending string ``s`` to child process, with
@@ -519,9 +517,11 @@ class spawn(SpawnBase):
         n = n + self.send(self.linesep)
         return n
 
-    def _log_control(self, byte):
+    def _log_control(self, s):
         """Write control characters to the appropriate log files"""
-        self._log(byte, 'send')
+        if self.encoding is not None:
+            s = s.decode(self.encoding, 'replace')
+        self._log(s, 'send')
 
     def sendcontrol(self, char):
         '''Helper method that wraps send() with mnemonic access for sending control
@@ -614,10 +614,17 @@ class spawn(SpawnBase):
         not read any data from the child, so this will block forever if the
         child has unread output and has terminated. In other words, the child
         may have printed output then called exit(), but, the child is
-        technically still alive until its output is read by the parent. '''
+        technically still alive until its output is read by the parent.
+
+        This method is non-blocking if :meth:`wait` has already been called
+        previously or :meth:`isalive` method returns False.  It simply returns
+        the previously determined exit status.
+        '''
 
         ptyproc = self.ptyproc
         with _wrap_ptyprocess_err():
+            # exception may occur if "Is some other process attempting
+            # "job control with our child pid?"
             exitstatus = ptyproc.wait()
         self.status = ptyproc.status
         self.exitstatus = ptyproc.exitstatus
@@ -677,11 +684,10 @@ class spawn(SpawnBase):
         the stdout and stderr output of the child process is printed. This
         simply echos the child stdout and child stderr to the real stdout and
         it echos the real stdin to the child stdin. When the user types the
-        escape_character this method will stop. The default for
-        escape_character is ^]. This should not be confused with ASCII 27 --
-        the ESC character. ASCII 29 was chosen for historical merit because
-        this is the character used by 'telnet' as the escape character. The
-        escape_character will not be sent to the child process.
+        escape_character this method will return None. The escape_character
+        will not be transmitted.  The default for escape_character is
+        entered as ``Ctrl - ]``, the very same as BSD telnet. To prevent
+        escaping, escape_character may be set to None.
 
         You may pass in optional input and output filter functions. These
         functions should take a string and return a string. The output_filter
@@ -713,7 +719,7 @@ class spawn(SpawnBase):
         self.buffer = self.string_type()
         mode = tty.tcgetattr(self.STDIN_FILENO)
         tty.setraw(self.STDIN_FILENO)
-        if PY3:
+        if escape_character is not None and PY3:
             escape_character = escape_character.encode('latin-1')
         try:
             self.__interact_copy(escape_character, input_filter, output_filter)
@@ -763,7 +769,9 @@ class spawn(SpawnBase):
                 data = self.__interact_read(self.STDIN_FILENO)
                 if input_filter:
                     data = input_filter(data)
-                i = data.rfind(escape_character)
+                i = -1
+                if escape_character is not None:
+                    i = data.rfind(escape_character)
                 if i != -1:
                     data = data[:i]
                     self.__interact_writen(self.child_fd, data)
@@ -798,22 +806,7 @@ class spawn(SpawnBase):
                     # this actually is an exception.
                     raise
 
-
-class spawnu(SpawnBaseUnicode, spawn):
-    """Works like spawn, but accepts and returns unicode strings.
-
-    Extra parameters:
-
-    :param encoding: The encoding to use for communications (default: 'utf-8')
-    :param errors: How to handle encoding/decoding errors; one of 'strict'
-                   (the default), 'ignore', or 'replace', as described
-                   for :meth:`~bytes.decode` and :meth:`~str.encode`.
-    """
-    ptyprocess_class = ptyprocess.PtyProcessUnicode
-
-    def _send(self, s):
-        return os.write(self.child_fd, s.encode(self.encoding, self.errors))
-
-    def _log_control(self, byte):
-        s = byte.decode(self.encoding, 'replace')
-        self._log(s, 'send')
+def spawnu(*args, **kwargs):
+    """Deprecated: pass encoding to spawn() instead."""
+    kwargs.setdefault('encoding', 'utf-8')
+    return spawn(*args, **kwargs)
diff --git a/pexpect/pxssh.py b/pexpect/pxssh.py
index 71f56a0..4638164 100644
--- a/pexpect/pxssh.py
+++ b/pexpect/pxssh.py
@@ -95,9 +95,12 @@ class pxssh (spawn):
 
     def __init__ (self, timeout=30, maxread=2000, searchwindowsize=None,
                     logfile=None, cwd=None, env=None, ignore_sighup=True, echo=True,
-                    options={}):
+                    options={}, encoding=None, codec_errors='strict'):
 
-        spawn.__init__(self, None, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, logfile=logfile, cwd=cwd, env=env, ignore_sighup=ignore_sighup, echo=echo)
+        spawn.__init__(self, None, timeout=timeout, maxread=maxread,
+                       searchwindowsize=searchwindowsize, logfile=logfile,
+                       cwd=cwd, env=env, ignore_sighup=ignore_sighup, echo=echo,
+                       encoding=encoding, codec_errors=codec_errors)
 
         self.name = '<pxssh>'
 
@@ -169,7 +172,7 @@ class pxssh (spawn):
         # maximum time for reading the entire prompt
         total_timeout = timeout_multiplier * 3.0
 
-        prompt = b''
+        prompt = self.string_type()
         begin = time.time()
         expired = 0.0
         timeout = first_char_timeout
@@ -334,7 +337,7 @@ class pxssh (spawn):
             if not self.set_unique_prompt():
                 self.close()
                 raise ExceptionPxssh('could not set shell prompt '
-                                     '(recieved: %r, expected: %r).' % (
+                                     '(received: %r, expected: %r).' % (
                                          self.before, self.PROMPT,))
         return True
 
diff --git a/pexpect/replwrap.py b/pexpect/replwrap.py
index 7b0e823..83a09c2 100644
--- a/pexpect/replwrap.py
+++ b/pexpect/replwrap.py
@@ -11,6 +11,7 @@ PY3 = (sys.version_info[0] >= 3)
 
 if PY3:
     def u(s): return s
+    basestring = str
 else:
     def u(s): return s.decode('utf-8')
 
@@ -37,7 +38,7 @@ class REPLWrapper(object):
                  new_prompt=PEXPECT_PROMPT,
                  continuation_prompt=PEXPECT_CONTINUATION_PROMPT,
                  extra_init_cmd=None):
-        if isinstance(cmd_or_spawn, str):
+        if isinstance(cmd_or_spawn, basestring):
             self.child = pexpect.spawnu(cmd_or_spawn, echo=False)
         else:
             self.child = cmd_or_spawn
diff --git a/pexpect/screen.py b/pexpect/screen.py
index efe9ee5..0bced89 100644
--- a/pexpect/screen.py
+++ b/pexpect/screen.py
@@ -27,6 +27,13 @@ import codecs
 import copy
 import sys
 
+import warnings
+
+warnings.warn(("pexpect.screen and pexpect.ANSI are deprecated. "
+               "We recommend using pyte to emulate a terminal screen: "
+               "https://pypi.python.org/pypi/pyte"),
+               stacklevel=2)
+
 NUL = 0    # Fill character; ignored on input.
 ENQ = 5    # Transmit answerback message.
 BEL = 7    # Ring the bell.
diff --git a/pexpect/spawnbase.py b/pexpect/spawnbase.py
index d79c5c0..9fd2e18 100644
--- a/pexpect/spawnbase.py
+++ b/pexpect/spawnbase.py
@@ -7,35 +7,30 @@ from .exceptions import ExceptionPexpect, EOF, TIMEOUT
 from .expect import Expecter, searcher_string, searcher_re
 
 PY3 = (sys.version_info[0] >= 3)
+text_type = str if PY3 else unicode
+
+class _NullCoder(object):
+    """Pass bytes through unchanged."""
+    @staticmethod
+    def encode(b, final=False):
+        return b
+
+    @staticmethod
+    def decode(b, final=False):
+        return b
 
 class SpawnBase(object):
     """A base class providing the backwards-compatible spawn API for Pexpect.
 
-    This should not be instantiated directly: use :class:`pexpect.spawn` or :class:`pexpect.fdpexpect.fdspawn`."""
-    string_type = bytes
-    if PY3:
-        allowed_string_types = (bytes, str)
-        linesep = os.linesep.encode('ascii')
-        crlf = '\r\n'.encode('ascii')
-
-        @staticmethod
-        def write_to_stdout(b):
-            try:
-                return sys.stdout.buffer.write(b)
-            except AttributeError:
-                # If stdout has been replaced, it may not have .buffer
-                return sys.stdout.write(b.decode('ascii', 'replace'))
-    else:
-        allowed_string_types = (basestring,)  # analysis:ignore
-        linesep = os.linesep
-        crlf = '\r\n'
-        write_to_stdout = sys.stdout.write
-
+    This should not be instantiated directly: use :class:`pexpect.spawn` or
+    :class:`pexpect.fdpexpect.fdspawn`.
+    """
     encoding = None
     pid = None
     flag_eof = False
 
-    def __init__(self, timeout=30, maxread=2000, searchwindowsize=None, logfile=None):
+    def __init__(self, timeout=30, maxread=2000, searchwindowsize=None,
+                 logfile=None, encoding=None, codec_errors='strict'):
         self.stdin = sys.stdin
         self.stdout = sys.stdout
         self.stderr = sys.stderr
@@ -63,7 +58,7 @@ class SpawnBase(object):
         # max bytes to read at one time into buffer
         self.maxread = maxread
         # This is the read buffer. See maxread.
-        self.buffer = self.string_type()
+        self.buffer = bytes() if (encoding is None) else text_type()
         # Data before searchwindowsize point is preserved, but not searched.
         self.searchwindowsize = searchwindowsize
         # Delay used before sending data to child. Time in seconds.
@@ -79,6 +74,42 @@ class SpawnBase(object):
         self.name = '<' + repr(self) + '>'
         self.closed = True
 
+        # Unicode interface
+        self.encoding = encoding
+        self.codec_errors = codec_errors
+        if encoding is None:
+            # bytes mode (accepts some unicode for backwards compatibility)
+            self._encoder = self._decoder = _NullCoder()
+            self.string_type = bytes
+            self.crlf = b'\r\n'
+            if PY3:
+                self.allowed_string_types = (bytes, str)
+                self.linesep = os.linesep.encode('ascii')
+                def write_to_stdout(b):
+                    try:
+                        return sys.stdout.buffer.write(b)
+                    except AttributeError:
+                        # If stdout has been replaced, it may not have .buffer
+                        return sys.stdout.write(b.decode('ascii', 'replace'))
+                self.write_to_stdout = write_to_stdout
+            else:
+                self.allowed_string_types = (basestring,)  # analysis:ignore
+                self.linesep = os.linesep
+                self.write_to_stdout = sys.stdout.write
+        else:
+            # unicode mode
+            self._encoder = codecs.getincrementalencoder(encoding)(codec_errors)
+            self._decoder = codecs.getincrementaldecoder(encoding)(codec_errors)
+            self.string_type = text_type
+            self.crlf = u'\r\n'
+            self.allowed_string_types = (text_type, )
+            if PY3:
+                self.linesep = os.linesep
+            else:
+                self.linesep = os.linesep.decode('ascii')
+            # This can handle unicode in both Python 2 and 3
+            self.write_to_stdout = sys.stdout.write
+
     def _log(self, s, direction):
         if self.logfile is not None:
             self.logfile.write(s)
@@ -88,22 +119,19 @@ class SpawnBase(object):
             second_log.write(s)
             second_log.flush()
 
-    @staticmethod
-    def _coerce_expect_string(s):
-        if not isinstance(s, bytes):
+    # For backwards compatibility, in bytes mode (when encoding is None)
+    # unicode is accepted for send and expect. Unicode mode is strictly unicode
+    # only.
+    def _coerce_expect_string(self, s):
+        if self.encoding is None and not isinstance(s, bytes):
             return s.encode('ascii')
         return s
 
-    @staticmethod
-    def _coerce_send_string(s):
-        if not isinstance(s, bytes):
+    def _coerce_send_string(self, s):
+        if self.encoding is None and not isinstance(s, bytes):
             return s.encode('utf-8')
         return s
 
-    @staticmethod
-    def _coerce_read_string(s):
-        return s
-
     def read_nonblocking(self, size=1, timeout=None):
         """This reads data from the file descriptor.
 
@@ -125,7 +153,7 @@ class SpawnBase(object):
             self.flag_eof = True
             raise EOF('End Of File (EOF). Empty string style platform.')
 
-        s = self._coerce_read_string(s)
+        s = self._decoder.decode(s, final=False)
         self._log(s, 'read')
         return s
 
@@ -451,34 +479,3 @@ class SpawnBase(object):
         # We rely on subclasses to implement close(). If they don't, it's not
         # clear what a context manager should do.
         self.close()
-
-class SpawnBaseUnicode(SpawnBase):
-    if PY3:
-        string_type = str
-        allowed_string_types = (str, )
-        linesep = os.linesep
-        crlf = '\r\n'
-    else:
-        string_type = unicode
-        allowed_string_types = (unicode, )
-        linesep = os.linesep.decode('ascii')
-        crlf = '\r\n'.decode('ascii')
-    # This can handle unicode in both Python 2 and 3
-    write_to_stdout = sys.stdout.write
-
-    def __init__(self, *args, **kwargs):
-        self.encoding = kwargs.pop('encoding', 'utf-8')
-        self.errors = kwargs.pop('errors', 'strict')
-        self._decoder = codecs.getincrementaldecoder(self.encoding)(errors=self.errors)
-        super(SpawnBaseUnicode, self).__init__(*args, **kwargs)
-
-    @staticmethod
-    def _coerce_expect_string(s):
-        return s
-
-    @staticmethod
-    def _coerce_send_string(s):
-        return s
-
-    def _coerce_read_string(self, s):
-        return self._decoder.decode(s, final=False)
\ No newline at end of file