merge heads

18 files changed, 139 insertions, 38 deletions

diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 2641c8b7f2..623bd7cea3 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -537,10 +537,11 @@ code, or when embedding the Python interpreter:
    operations such as ``PyEval_ReleaseThread(tstate)``. It is not needed before
    calling :c:func:`PyEval_SaveThread` or :c:func:`PyEval_RestoreThread`.
 
-   .. index:: single: Py_Initialize()
-
    This is a no-op when called for a second time.
 
+   .. versionchanged:: 3.2
+      This function cannot be called before :c:func:`Py_Initialize()` anymore.
+
    .. index:: module: _thread
 
    .. note::
diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst
index 11984a5364..ebe2d9cb47 100644
--- a/Doc/library/email.message.rst
+++ b/Doc/library/email.message.rst
@@ -46,15 +46,16 @@ Here are the methods of the :class:`Message` class:
       be generated or modified).
 
       Note that this method is provided as a convenience and may not always
-      format the message the way you want.  For example, by default it mangles
-      lines that begin with ``From``.  For more flexibility, instantiate a
+      format the message the way you want.  For example, by default it does
+      not do the mangling of lines that begin with ``From`` that is
+      required by the unix mbox format.  For more flexibility, instantiate a
       :class:`~email.generator.Generator` instance and use its :meth:`flatten`
       method directly.  For example::
 
          from io import StringIO
          from email.generator import Generator
          fp = StringIO()
-         g = Generator(fp, mangle_from_=False, maxheaderlen=60)
+         g = Generator(fp, mangle_from_=True, maxheaderlen=60)
          g.flatten(msg)
          text = fp.getvalue()
 
diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index 531a64d73f..4805c8e205 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -34,6 +34,20 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions).
    For normal use, you should only require the initialization/connect,
    :meth:`sendmail`, and :meth:`quit` methods.  An example is included below.
 
+   The :class:`SMTP` class supports the :keyword:`with` statement.  When used
+   like this, the SMTP ``QUIT`` command is issued automatically when the
+   :keyword:`with` statement exits.  E.g.::
+
+    >>> from smtplib import SMTP
+    >>> with SMTP("domain.org") as smtp:
+    ...     smtp.noop()
+    ...
+    (250, b'Ok')
+    >>>
+
+   .. versionadded:: 3.3
+      Support for the :keyword:`with` statement was added.
+
 
 .. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, certfile=None[, timeout])
 
diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst
index 1baa9e1470..256b343767 100644
--- a/Doc/whatsnew/3.2.rst
+++ b/Doc/whatsnew/3.2.rst
@@ -2698,3 +2698,7 @@ require changes to your code:
   a new function, :func:`asyncore.handle_accepted`, was added to replace it.
 
   (Contributed by Giampaolo Rodola in :issue:`6706`.)
+
+* Due to the new :term:`GIL` implementation, :c:func:`PyEval_InitThreads()`
+  cannot be called before :c:func:`Py_Initialize()` anymore.
+
diff --git a/Lib/email/charset.py b/Lib/email/charset.py
index 24d5545127..f22be2c52c 100644
--- a/Lib/email/charset.py
+++ b/Lib/email/charset.py
@@ -263,7 +263,7 @@ class Charset:
 
         Returns "quoted-printable" if self.body_encoding is QP.
         Returns "base64" if self.body_encoding is BASE64.
-        Returns "7bit" otherwise.
+        Returns conversion function otherwise.
         """
         assert self.body_encoding != SHORTEST
         if self.body_encoding == QP:
@@ -381,7 +381,10 @@ class Charset:
         """Body-encode a string by converting it first to bytes.
 
         The type of encoding (base64 or quoted-printable) will be based on
-        self.body_encoding.
+        self.body_encoding.  If body_encoding is None, we assume the
+        output charset is a 7bit encoding, so re-encoding the decoded
+        string using the ascii codec produces the correct string version
+        of the content.
         """
         # 7bit/8bit encodings return the string unchanged (module conversions)
         if self.body_encoding is BASE64:
@@ -391,4 +394,6 @@ class Charset:
         elif self.body_encoding is QP:
             return email.quoprimime.body_encode(string)
         else:
+            if isinstance(string, str):
+                string = string.encode(self.output_charset).decode('ascii')
             return string
diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py
index 0ea441d963..c66f4ccf38 100644
--- a/Lib/email/encoders.py
+++ b/Lib/email/encoders.py
@@ -54,10 +54,13 @@ def encode_7or8bit(msg):
         # There's no payload.  For backwards compatibility we use 7bit
         msg['Content-Transfer-Encoding'] = '7bit'
         return
-    # We play a trick to make this go fast.  If encoding to ASCII succeeds, we
-    # know the data must be 7bit, otherwise treat it as 8bit.
+    # We play a trick to make this go fast.  If encoding/decode to ASCII
+    # succeeds, we know the data must be 7bit, otherwise treat it as 8bit.
     try:
-        orig.encode('ascii')
+        if isinstance(orig, str):
+            orig.encode('ascii')
+        else:
+            orig.decode('ascii')
     except UnicodeError:
         # iso-2022-* is non-ASCII but still 7-bit
         charset = msg.get_charset()
diff --git a/Lib/email/message.py b/Lib/email/message.py
index b821bfd622..2713bc5d35 100644
--- a/Lib/email/message.py
+++ b/Lib/email/message.py
@@ -157,8 +157,7 @@ class Message:
         header.
 
         This is a convenience method and may not generate the message exactly
-        as you intend because by default it mangles lines that begin with
-        "From ".  For more flexibility, use the flatten() method of a
+        as you intend.  For more flexibility, use the flatten() method of a
         Generator instance.
         """
         from email.generator import Generator
@@ -242,7 +241,7 @@ class Message:
             raise TypeError('Expected list, got %s' % type(self._payload))
         payload = self._payload
         cte = self.get('content-transfer-encoding', '').lower()
-        # payload can be bytes here, (I wonder if that is actually a bug?)
+        # payload may be bytes here.
         if isinstance(payload, str):
             if _has_surrogates(payload):
                 bpayload = payload.encode('ascii', 'surrogateescape')
diff --git a/Lib/email/test/test_email.py b/Lib/email/test/test_email.py
index f2d5210727..3fa8f9326e 100644
--- a/Lib/email/test/test_email.py
+++ b/Lib/email/test/test_email.py
@@ -3372,9 +3372,9 @@ class TestCharset(unittest.TestCase):
         # built-in encodings where the header encoding is QP but the body
         # encoding is not.
         from email import charset as CharsetModule
-        CharsetModule.add_charset('fake', CharsetModule.QP, None)
+        CharsetModule.add_charset('fake', CharsetModule.QP, None, 'utf-8')
         c = Charset('fake')
-        eq('hello w\xf6rld', c.body_encode('hello w\xf6rld'))
+        eq('hello world', c.body_encode('hello world'))
 
     def test_unicode_charset_name(self):
         charset = Charset('us-ascii')
diff --git a/Lib/email/test/test_email_codecs.py b/Lib/email/test/test_email_codecs.py
index acc19c30f9..ca85f5731e 100644
--- a/Lib/email/test/test_email_codecs.py
+++ b/Lib/email/test/test_email_codecs.py
@@ -13,7 +13,7 @@ from email.message import Message
 # We're compatible with Python 2.3, but it doesn't have the built-in Asian
 # codecs, so we have to skip all these tests.
 try:
-    str('foo', 'euc-jp')
+    str(b'foo', 'euc-jp')
 except LookupError:
     raise unittest.SkipTest
 
@@ -22,11 +22,14 @@ except LookupError:
 class TestEmailAsianCodecs(TestEmailBase):
     def test_japanese_codecs(self):
         eq = self.ndiffAssertEqual
-        j = Charset("euc-jp")
-        g = Charset("iso-8859-1")
+        jcode = "euc-jp"
+        gcode = "iso-8859-1"
+        j = Charset(jcode)
+        g = Charset(gcode)
         h = Header("Hello World!")
-        jhello = '\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc\xa5\xeb\xa5\xc9\xa1\xaa'
-        ghello = 'Gr\xfc\xdf Gott!'
+        jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc'
+                     b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode)
+        ghello = str(b'Gr\xfc\xdf Gott!', gcode)
         h.append(jhello, j)
         h.append(ghello, g)
         # BAW: This used to -- and maybe should -- fold the two iso-8859-1
@@ -36,13 +39,17 @@ class TestEmailAsianCodecs(TestEmailBase):
         # encoded word.
         eq(h.encode(), """\
 Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?=
- =?iso-8859-1?q?Gr=FC=DF?= =?iso-8859-1?q?_Gott!?=""")
+ =?iso-8859-1?q?Gr=FC=DF_Gott!?=""")
         eq(decode_header(h.encode()),
-           [('Hello World!', None),
-            ('\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'),
-            ('Gr\xfc\xdf Gott!', 'iso-8859-1')])
-        int = 'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9'
-        h = Header(int, j, header_name="Subject")
+           [(b'Hello World!', None),
+            (b'\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'),
+            (b'Gr\xfc\xdf Gott!', gcode)])
+        subject_bytes = (b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5'
+            b'\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2'
+            b'\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3'
+            b'\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9')
+        subject = str(subject_bytes, jcode)
+        h = Header(subject, j, header_name="Subject")
         # test a very long header
         enc = h.encode()
         # TK: splitting point may differ by codec design and/or Header encoding
@@ -50,15 +57,24 @@ Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?=
 =?iso-2022-jp?b?dGVzdC1qYSAbJEIkWEVqOUYkNSRsJD8lYSE8JWskTztKGyhC?=
  =?iso-2022-jp?b?GyRCMnE8VCROPjVHJyRyQlQkQyRGJCQkXiQ5GyhC?=""")
         # TK: full decode comparison
-        eq(h.__unicode__().encode('euc-jp'), int)
+        eq(str(h).encode(jcode), subject_bytes)
+
+    def test_payload_encoding_utf8(self):
+        jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc'
+                     b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp')
+        msg = Message()
+        msg.set_payload(jhello, 'utf-8')
+        ustr = msg.get_payload(decode=True).decode(msg.get_content_charset())
+        self.assertEqual(jhello, ustr)
 
     def test_payload_encoding(self):
-        jhello = '\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc\xa5\xeb\xa5\xc9\xa1\xaa'
         jcode  = 'euc-jp'
+        jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc'
+                     b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode)
         msg = Message()
         msg.set_payload(jhello, jcode)
-        ustr = str(msg.get_payload(), msg.get_content_charset())
-        self.assertEqual(jhello, ustr.encode(jcode))
+        ustr = msg.get_payload(decode=True).decode(msg.get_content_charset())
+        self.assertEqual(jhello, ustr)
 
 
 
diff --git a/Lib/smtplib.py b/Lib/smtplib.py
index 14e62504d3..213138c86e 100755
--- a/Lib/smtplib.py
+++ b/Lib/smtplib.py
@@ -269,6 +269,19 @@ class SMTP:
                     pass
                 self.local_hostname = '[%s]' % addr
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        try:
+            code, message = self.docmd("QUIT")
+            if code != 221:
+                raise SMTPResponseException(code, message)
+        except SMTPServerDisconnected:
+            pass
+        finally:
+            self.close()
+
     def set_debuglevel(self, debuglevel):
         """Set the debug output level.
 
diff --git a/Lib/test/test_email.py b/Lib/test/test_email.py
index f0a18e71d0..5eebba5590 100644
--- a/Lib/test/test_email.py
+++ b/Lib/test/test_email.py
@@ -3,10 +3,12 @@
 
 # The specific tests now live in Lib/email/test
 from email.test.test_email import suite
+from email.test.test_email_codecs import suite as codecs_suite
 from test import support
 
 def test_main():
     support.run_unittest(suite())
+    support.run_unittest(codecs_suite())
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index 4651f376f6..d973faafcc 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -424,6 +424,9 @@ sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'],
 # Simulated SMTP channel & server
 class SimSMTPChannel(smtpd.SMTPChannel):
 
+    # For testing failures in QUIT when using the context manager API.
+    quit_response = None
+
     def __init__(self, extra_features, *args, **kw):
         self._extrafeatures = ''.join(
             [ "250-{0}\r\n".format(x) for x in extra_features ])
@@ -475,19 +478,31 @@ class SimSMTPChannel(smtpd.SMTPChannel):
         else:
             self.push('550 No access for you!')
 
+    def smtp_QUIT(self, arg):
+        # args is ignored
+        if self.quit_response is None:
+            super(SimSMTPChannel, self).smtp_QUIT(arg)
+        else:
+            self.push(self.quit_response)
+            self.close_when_done()
+
     def handle_error(self):
         raise
 
 
 class SimSMTPServer(smtpd.SMTPServer):
 
+    # For testing failures in QUIT when using the context manager API.
+    quit_response = None
+
     def __init__(self, *args, **kw):
         self._extra_features = []
         smtpd.SMTPServer.__init__(self, *args, **kw)
 
     def handle_accepted(self, conn, addr):
-        self._SMTPchannel = SimSMTPChannel(self._extra_features,
-                                           self, conn, addr)
+        self._SMTPchannel = SimSMTPChannel(
+            self._extra_features, self, conn, addr)
+        self._SMTPchannel.quit_response = self.quit_response
 
     def process_message(self, peer, mailfrom, rcpttos, data):
         pass
@@ -620,6 +635,25 @@ class SMTPSimTests(unittest.TestCase):
             self.assertIn(sim_auth_credentials['cram-md5'], str(err))
         smtp.close()
 
+    def test_with_statement(self):
+        with smtplib.SMTP(HOST, self.port) as smtp:
+            code, message = smtp.noop()
+            self.assertEqual(code, 250)
+        self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo')
+        with smtplib.SMTP(HOST, self.port) as smtp:
+            smtp.close()
+        self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo')
+
+    def test_with_statement_QUIT_failure(self):
+        self.serv.quit_response = '421 QUIT FAILED'
+        with self.assertRaises(smtplib.SMTPResponseException) as error:
+            with smtplib.SMTP(HOST, self.port) as smtp:
+                smtp.noop()
+        self.assertEqual(error.exception.smtp_code, 421)
+        self.assertEqual(error.exception.smtp_error, b'QUIT FAILED')
+        # We don't need to clean up self.serv.quit_response because a new
+        # server is always instantiated in the setUp().
+
     #TODO: add tests for correct AUTH method fallback now that the
     #test infrastructure can support it.
 
diff --git a/Misc/ACKS b/Misc/ACKS
index d0b8951c96..c9982500ad 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -376,6 +376,7 @@ Kevan Heydon
 Jason Hildebrand
 Richie Hindle
 Konrad Hinsen
+Michael Henry
 David Hobley
 Tim Hochberg
 Joerg-Cyril Hoehle
diff --git a/Misc/NEWS b/Misc/NEWS
index f9c4c9e1f4..018c62d8c8 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -68,6 +68,12 @@ Core and Builtins
 Library
 -------
 
+- Issue #11289: `smtp.SMTP` class becomes a context manager so it can be used
+  in a `with` statement.  Contributed by Giampaolo Rodola.
+
+- Issue #11554: Fixed support for Japanese codecs; previously the body output
+  encoding was not done if euc-jp or shift-jis was specified as the charset.
+
 - Issue #11509: Significantly increase test coverage of fileinput.
   Patch by Denver Coneybeare at PyCon 2011 Sprints.
 
@@ -206,6 +212,8 @@ Tools/Demos
 Tests
 -----
 
+- Issue #11554: Reactivated test_email_codecs.
+
 - Issue #11505: improves test coverage of string.py
 
 - Issue #11490: test_subprocess:test_leaking_fds_on_error no longer gives a
diff --git a/Modules/_ctypes/libffi/ChangeLog b/Modules/_ctypes/libffi/ChangeLog
index 78853c42f5..00ba7efc1d 100644
--- a/Modules/_ctypes/libffi/ChangeLog
+++ b/Modules/_ctypes/libffi/ChangeLog
@@ -3084,7 +3084,7 @@
 
 2003-09-18  Kaz Kojima  <kkojima@gcc.gnu.org>
 
-	* src/sh/ffi.c (ffi_prep_args): Take account into the alignment
+	* src/sh/ffi.c (ffi_prep_args): Take account into the alignement
 	for the register size.
 	(ffi_closure_helper_SYSV): Handle the structure return value
 	address correctly.
@@ -3344,7 +3344,7 @@
 2003-02-06  Andreas Tobler <a.tobler@schweiz.ch>
 
 	* libffi/src/powerpc/darwin_closure.S:
-	Fix alignment bug, allocate 8 bytes for the result.
+	Fix alignement bug, allocate 8 bytes for the result.
 	* libffi/src/powerpc/aix_closure.S:
 	Likewise.
 	* libffi/src/powerpc/ffi_darwin.c:
diff --git a/Modules/_ctypes/libffi/src/dlmalloc.c b/Modules/_ctypes/libffi/src/dlmalloc.c
index 72673a1bc7..783c5c2a6f 100644
--- a/Modules/_ctypes/libffi/src/dlmalloc.c
+++ b/Modules/_ctypes/libffi/src/dlmalloc.c
@@ -1326,7 +1326,7 @@ static void* win32direct_mmap(size_t size) {
   return (ptr != 0)? ptr: MFAIL;
 }
 
-/* This function supports releasing coalesced segments */
+/* This function supports releasing coalesed segments */
 static int win32munmap(void* ptr, size_t size) {
   MEMORY_BASIC_INFORMATION minfo;
   char* cptr = ptr;
@@ -1362,7 +1362,7 @@ static int win32munmap(void* ptr, size_t size) {
 #define CALL_MORECORE(S)     MFAIL
 #endif /* HAVE_MORECORE */
 
-/* mstate bit set if contiguous morecore disabled or failed */
+/* mstate bit set if continguous morecore disabled or failed */
 #define USE_NONCONTIGUOUS_BIT (4U)
 
 /* segment bit set in create_mspace_with_base */
diff --git a/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c b/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c
index 5e7772068a..d84f1c393a 100644
--- a/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c
+++ b/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c
@@ -592,7 +592,7 @@ typedef struct aix_fd_struct {
 		  +---------------------------------------+ 160
 		  | result area 8                         |
 		  +---------------------------------------+ 168
-		  | alignment to the next multiple of 16  |
+		  | alignement to the next multiple of 16 |
 SP current -->    +---------------------------------------+ 176 <- parent frame
 		  | back chain to caller 4                |
 		  +---------------------------------------+ 180
diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c
index dba2184099..8953d5fda3 100644
--- a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c
+++ b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c
@@ -650,7 +650,7 @@ ffi_call(
 		  +---------------------------------------+ 160
 		  | result area 8                         |
 		  +---------------------------------------+ 168
-		  | alignment to the next multiple of 16  |
+		  | alignement to the next multiple of 16 |
 SP current -->    +---------------------------------------+ 176 <- parent frame
 		  | back chain to caller 4                |
 		  +---------------------------------------+ 180