#21795: advertise 8BITMIME if decode_data is False.

Patch by Milan Oberkirch, with a few updates.  This changeset also
tweaks the smtpd and whatsnew docs for smtpd into what should be
the final form for the 3.5 release.

5 files changed, 219 insertions, 97 deletions

diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst
index 3e0c6fbff5..575dcec994 100644
--- a/Doc/library/smtpd.rst
+++ b/Doc/library/smtpd.rst
@@ -40,20 +40,27 @@ SMTPServer Objects
    accepted in a ``DATA`` command.  A value of ``None`` or ``0`` means no
    limit.
 
-   *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined
-   in :RFC:`6531`) should be enabled.  The default is ``False``.  If
-   *enable_SMTPUTF* is set to ``True``, the :meth:`process_smtputf8_message`
-   method must be defined.  A :exc:`ValueError` is raised if both
-   *enable_SMTPUTF8* and *decode_data* are set to ``True`` at the same time.
+   *map* is the socket map to use for connections (an initially empty
+   dictionary is a suitable value).  If not specified the :mod:`asyncore`
+   global socket map is used.
 
-   A dictionary can be specified in *map* to avoid using a global socket map.
+   *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined
+   in :RFC:`6531`) should be enabled.  The default is ``False``.  If set to
+   ``True``, *decode_data* must be ``False`` (otherwise an error is raised).
+   When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL``
+   command and when present is passed to :meth:`process_message` in the
+   ``kwargs['mail_options']`` list.
 
    *decode_data* specifies whether the data portion of the SMTP transaction
    should be decoded using UTF-8.  The default is ``True`` for backward
-   compatibility reasons, but will change to ``False`` in Python 3.6.  Specify
-   the keyword value explicitly to avoid the :exc:`DeprecationWarning`.
+   compatibility reasons, but will change to ``False`` in Python 3.6; specify
+   the keyword value explicitly to avoid the :exc:`DeprecationWarning`.  When
+   *decode_data* is set to ``False`` the server advertises the ``8BITMIME``
+   extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to
+   the ``MAIL`` command, and when present passes it to :meth:`process_message`
+   in the ``kwargs['mail_options']`` list.
 
-   .. method:: process_message(peer, mailfrom, rcpttos, data)
+   .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs)
 
       Raise a :exc:`NotImplementedError` exception. Override this in subclasses to
       do something useful with this message. Whatever was passed in the
@@ -67,34 +74,39 @@ SMTPServer Objects
       argument will be a unicode string.  If it is set to ``False``, it
       will be a bytes object.
 
-      Return ``None`` to request a normal ``250 Ok`` response; otherwise
-      return the desired response string in :RFC:`5321` format.
+      *kwargs* is a dictionary containing additional information. It is empty
+      unless at least one of ``decode_data=False`` or ``enable_SMTPUTF8=True``
+      was given as an init parameter, in which case it contains the following
+      keys:
+
+          *mail_options*:
+             a list of all received parameters to the ``MAIL``
+             command (the elements are uppercase strings; example:
+             ``['BODY=8BITMIME', 'SMTPUTF8']``).
 
-   .. method:: process_smtputf8_message(peer, mailfrom, rcpttos, data)
+          *rcpt_options*:
+             same as *mail_options* but for the ``RCPT`` command.
+             Currently no ``RCPT TO`` options are supported, so for now
+             this will always be an empty list.
 
-      Raise a :exc:`NotImplementedError` exception.  Override this in
-      subclasses to do something useful with messages when *enable_SMTPUTF8*
-      has been set to ``True`` and the SMTP client requested ``SMTPUTF8``,
-      since this method is called rather than :meth:`process_message` when the
-      client actively requests ``SMTPUTF8``.  The *data* argument will always
-      be a bytes object, and any non-``None`` return value should conform to
-      :rfc:`6531`; otherwise, the API is the same as for
-      :meth:`process_message`.
+      Return ``None`` to request a normal ``250 Ok`` response; otherwise
+      return the desired response string in :RFC:`5321` format.
 
    .. attribute:: channel_class
 
       Override this in subclasses to use a custom :class:`SMTPChannel` for
       managing SMTP clients.
 
-   .. versionchanged:: 3.4
-      The *map* argument was added.
+   .. versionadded:: 3.4
+      The *map* constructor argument.
 
    .. versionchanged:: 3.5
       *localaddr* and *remoteaddr* may now contain IPv6 addresses.
 
    .. versionadded:: 3.5
       the *decode_data* and *enable_SMTPUTF8* constructor arguments, and the
-      :meth:`process_smtputf8_message` method.
+      *kwargs* argument to :meth:`process_message` when one or more of these is
+      specified.
 
 
 DebuggingServer Objects
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
index 02a7065a37..86febb0691 100644
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -468,16 +468,28 @@ smtpd
   transaction is decoded using the ``utf-8`` codec or is instead provided to
   :meth:`~smtpd.SMTPServer.process_message` as a byte string.  The default
   is ``True`` for backward compatibility reasons, but will change to ``False``
-  in Python 3.6.  (Contributed by Maciej Szulik in :issue:`19662`.)
+  in Python 3.6.  If *decode_data* is set to ``False``, the
+  :meth:`~smtpd.SMTPServer.process_message` method must be prepared to accept
+  keyword arguments.  (Contributed by Maciej Szulik in :issue:`19662`.)
+
+* :class:`~smtpd.SMTPServer` now advertises the ``8BITMIME`` extension
+  (:rfc:`6152`) if if *decode_data* has been set ``True``.  If the client
+  specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to
+  :meth:`~smtpd.SMTPServer.process_message` via the ``mail_options`` keyword.
+  (Contributed by Milan Oberkirch and R.  David Murray in :issue:`21795`.)
+
+* :class:`~smtpd.SMTPServer` now supports the ``SMTPUTF8`` extension
+  (:rfc:`6531`: Internationalized Email).  If the client specified ``SMTPUTF8
+  BODY=8BITMIME`` on the ``MAIL`` command, they are passed to
+  :meth:`~smtpd.SMTPServer.process_message` via the ``mail_options`` keyword.
+  It is the responsibility of the :meth:`~smtpd.SMTPServer.process_message`
+  method to correctly handle the ``SMTPUTF8`` data.  (Contributed by Milan
+  Oberkirch in :issue:`21725`.)
 
 * It is now possible to provide, directly or via name resolution, IPv6
   addresses in the :class:`~smtpd.SMTPServer` constructor, and have it
   successfully connect.  (Contributed by Milan Oberkirch in :issue:`14758`.)
 
-* :mod:`~smtpd.SMTPServer` now supports :rfc:`6531` via the *enable_SMTPUTF8*
-  constructor argument and a user-provided
-  :meth:`~smtpd.SMTPServer.process_smtputf8_message` method.
-
 smtplib
 -------
 
diff --git a/Lib/smtpd.py b/Lib/smtpd.py
index dd410b8e16..ff86e7d206 100755
--- a/Lib/smtpd.py
+++ b/Lib/smtpd.py
@@ -381,10 +381,13 @@ class SMTPChannel(asynchat.async_chat):
                     data.append(text)
             self.received_data = self._newline.join(data)
             args = (self.peer, self.mailfrom, self.rcpttos, self.received_data)
-            if self.require_SMTPUTF8:
-                status = self.smtp_server.process_smtputf8_message(*args)
-            else:
-                status = self.smtp_server.process_message(*args)
+            kwargs = {}
+            if not self._decode_data:
+                kwargs = {
+                    'mail_options': self.mail_options,
+                    'rcpt_options': self.rcpt_options,
+                }
+            status = self.smtp_server.process_message(*args, **kwargs)
             self._set_post_data_state()
             if not status:
                 self.push('250 OK')
@@ -419,8 +422,9 @@ class SMTPChannel(asynchat.async_chat):
         if self.data_size_limit:
             self.push('250-SIZE %s' % self.data_size_limit)
             self.command_size_limits['MAIL'] += 26
-        if self.enable_SMTPUTF8:
+        if not self._decode_data:
             self.push('250-8BITMIME')
+        if self.enable_SMTPUTF8:
             self.push('250-SMTPUTF8')
             self.command_size_limits['MAIL'] += 10
         self.push('250 HELP')
@@ -454,11 +458,15 @@ class SMTPChannel(asynchat.async_chat):
         return address.addr_spec, rest
 
     def _getparams(self, params):
-        # Return any parameters that appear to be syntactically valid according
-        # to RFC 1869, ignore all others.  (Postel rule: accept what we can.)
-        params = [param.split('=', 1) if '=' in param else (param, True)
-                  for param in params.split()]
-        return {k: v for k, v in params if k.isalnum()}
+        # Return params as dictionary. Return None if not all parameters
+        # appear to be syntactically valid according to RFC 1869.
+        result = {}
+        for param in params:
+            param, eq, value = param.partition('=')
+            if not param.isalnum() or eq and not value:
+                return None
+            result[param] = value if eq else True
+        return result
 
     def smtp_HELP(self, arg):
         if arg:
@@ -508,7 +516,7 @@ class SMTPChannel(asynchat.async_chat):
 
     def smtp_MAIL(self, arg):
         if not self.seen_greeting:
-            self.push('503 Error: send HELO first');
+            self.push('503 Error: send HELO first')
             return
         print('===> MAIL', arg, file=DEBUGSTREAM)
         syntaxerr = '501 Syntax: MAIL FROM: <address>'
@@ -528,18 +536,23 @@ class SMTPChannel(asynchat.async_chat):
         if self.mailfrom:
             self.push('503 Error: nested MAIL command')
             return
-        params = self._getparams(params.upper())
+        self.mail_options = params.upper().split()
+        params = self._getparams(self.mail_options)
         if params is None:
             self.push(syntaxerr)
             return
-        body = params.pop('BODY', '7BIT')
-        if self.enable_SMTPUTF8 and params.pop('SMTPUTF8', False):
-            if body != '8BITMIME':
-                self.push('501 Syntax: MAIL FROM: <address>'
-                          ' [BODY=8BITMIME SMTPUTF8]')
+        if not self._decode_data:
+            body = params.pop('BODY', '7BIT')
+            if body not in ['7BIT', '8BITMIME']:
+                self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME')
                 return
-            else:
+        if self.enable_SMTPUTF8:
+            smtputf8 = params.pop('SMTPUTF8', False)
+            if smtputf8 is True:
                 self.require_SMTPUTF8 = True
+            elif smtputf8 is not False:
+                self.push('501 Error: SMTPUTF8 takes no arguments')
+                return
         size = params.pop('SIZE', None)
         if size:
             if not size.isdigit():
@@ -574,16 +587,16 @@ class SMTPChannel(asynchat.async_chat):
         if not address:
             self.push(syntaxerr)
             return
-        if params:
-            if self.extended_smtp:
-                params = self._getparams(params.upper())
-                if params is None:
-                    self.push(syntaxerr)
-                    return
-            else:
-                self.push(syntaxerr)
-                return
-        if params and len(params.keys()) > 0:
+        if not self.extended_smtp and params:
+            self.push(syntaxerr)
+            return
+        self.rcpt_options = params.upper().split()
+        params = self._getparams(self.rcpt_options)
+        if params is None:
+            self.push(syntaxerr)
+            return
+        # XXX currently there are no options we recognize.
+        if len(params.keys()) > 0:
             self.push('555 RCPT TO parameters not recognized or not implemented')
             return
         self.rcpttos.append(address)
@@ -667,7 +680,7 @@ class SMTPServer(asyncore.dispatcher):
                                      self._decode_data)
 
     # API for "doing something useful with the message"
-    def process_message(self, peer, mailfrom, rcpttos, data):
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
         """Override this abstract method to handle messages from the client.
 
         peer is a tuple containing (ipaddr, port) of the client that made the
@@ -685,21 +698,16 @@ class SMTPServer(asyncore.dispatcher):
         containing a `.' followed by other text has had the leading dot
         removed.
 
-        This function should return None for a normal `250 Ok' response;
-        otherwise, it should return the desired response string in RFC 821
-        format.
-
-        """
-        raise NotImplementedError
-
-    # API for processing messeges needing Unicode support (RFC 6531, RFC 6532).
-    def process_smtputf8_message(self, peer, mailfrom, rcpttos, data):
-        """Same as ``process_message`` but for messages for which the client
-        has sent the SMTPUTF8 parameter with the MAIL command (see the
-        enable_SMTPUTF8 parameter of the constructor).
+        kwargs is a dictionary containing additional information. It is empty
+        unless decode_data=False or enable_SMTPUTF8=True was given as init
+        parameter, in which case ut will contain the following keys:
+            'mail_options': list of parameters to the mail command.  All
+                            elements are uppercase strings.  Example:
+                            ['BODY=8BITMIME', 'SMTPUTF8'].
+            'rcpt_options': same, for the rcpt command.
 
         This function should return None for a normal `250 Ok' response;
-        otherwise, it should return the desired response string in RFC 6531
+        otherwise, it should return the desired response string in RFC 821
         format.
 
         """
@@ -725,13 +733,13 @@ class DebuggingServer(SMTPServer):
                 line = repr(line)
             print(line)
 
-    def process_message(self, peer, mailfrom, rcpttos, data):
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
         print('---------- MESSAGE FOLLOWS ----------')
-        self._print_message_content(peer, data)
-        print('------------ END MESSAGE ------------')
-
-    def process_smtputf8_message(self, peer, mailfrom, rcpttos, data):
-        print('----- SMTPUTF8 MESSAGE FOLLOWS ------')
+        if kwargs:
+            if kwargs.get('mail_options'):
+                print('mail options: %s' % kwargs['mail_options'])
+            if kwargs.get('rcpt_options'):
+                print('rcpt options: %s\n' % kwargs['rcpt_options'])
         self._print_message_content(peer, data)
         print('------------ END MESSAGE ------------')
 
diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py
index 6eb47f1dee..1aa55d2144 100644
--- a/Lib/test/test_smtpd.py
+++ b/Lib/test/test_smtpd.py
@@ -16,13 +16,12 @@ class DummyServer(smtpd.SMTPServer):
         else:
             self.return_status = b'return status'
 
-    def process_message(self, peer, mailfrom, rcpttos, data):
+    def process_message(self, peer, mailfrom, rcpttos, data, **kw):
         self.messages.append((peer, mailfrom, rcpttos, data))
         if data == self.return_status:
             return '250 Okish'
-
-    def process_smtputf8_message(self, *args, **kwargs):
-        return '250 SMTPUTF8 message okish'
+        if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']:
+            return '250 SMTPUTF8 message okish'
 
 
 class DummyDispatcherBroken(Exception):
@@ -54,22 +53,6 @@ class SMTPDServerTest(unittest.TestCase):
         write_line(b'DATA')
         self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n')
 
-    def test_process_smtputf8_message_unimplemented(self):
-        server = smtpd.SMTPServer((support.HOST, 0), ('b', 0),
-                                  enable_SMTPUTF8=True)
-        conn, addr = server.accept()
-        channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True)
-
-        def write_line(line):
-            channel.socket.queue_recv(line)
-            channel.handle_read()
-
-        write_line(b'EHLO example')
-        write_line(b'MAIL From: <eggs@example> BODY=8BITMIME SMTPUTF8')
-        write_line(b'RCPT To: <spam@example>')
-        write_line(b'DATA')
-        self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n')
-
     def test_decode_data_default_warns(self):
         with self.assertWarns(DeprecationWarning):
             smtpd.SMTPServer((support.HOST, 0), ('b', 0))
@@ -168,7 +151,8 @@ class DebuggingServerTest(unittest.TestCase):
                            enable_SMTPUTF8=True)
         stdout = s.getvalue()
         self.assertEqual(stdout, textwrap.dedent("""\
-             ----- SMTPUTF8 MESSAGE FOLLOWS ------
+             ---------- MESSAGE FOLLOWS ----------
+             mail options: ['BODY=8BITMIME', 'SMTPUTF8']
              b'From: test'
              b'X-Peer: peer-address'
              b''
@@ -201,6 +185,109 @@ class TestFamilyDetection(unittest.TestCase):
         self.assertEqual(server.socket.family, socket.AF_INET)
 
 
+class TestRcptOptionParsing(unittest.TestCase):
+    error_response = (b'555 RCPT TO parameters not recognized or not '
+                      b'implemented\r\n')
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, channel, line):
+        channel.socket.queue_recv(line)
+        channel.handle_read()
+
+    def test_params_rejected(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(channel, b'MAIL from: <foo@example.com> size=20')
+        self.write_line(channel, b'RCPT to: <foo@example.com> foo=bar')
+        self.assertEqual(channel.socket.last, self.error_response)
+
+    def test_nothing_accepted(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(channel, b'MAIL from: <foo@example.com> size=20')
+        self.write_line(channel, b'RCPT to: <foo@example.com>')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+
+class TestMailOptionParsing(unittest.TestCase):
+    error_response = (b'555 MAIL FROM parameters not recognized or not '
+                      b'implemented\r\n')
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, channel, line):
+        channel.socket.queue_recv(line)
+        channel.handle_read()
+
+    def test_with_decode_data_true(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True)
+        self.write_line(channel, b'EHLO example')
+        for line in [
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8',
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=8BITMIME',
+            b'MAIL from: <foo@example.com> size=20 BODY=UNKNOWN',
+            b'MAIL from: <foo@example.com> size=20 body=8bitmime',
+        ]:
+            self.write_line(channel, line)
+            self.assertEqual(channel.socket.last, self.error_response)
+        self.write_line(channel, b'MAIL from: <foo@example.com> size=20')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+    def test_with_decode_data_false(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False)
+        self.write_line(channel, b'EHLO example')
+        for line in [
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8',
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=8BITMIME',
+        ]:
+            self.write_line(channel, line)
+            self.assertEqual(channel.socket.last, self.error_response)
+        self.write_line(
+            channel,
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=UNKNOWN')
+        self.assertEqual(
+            channel.socket.last,
+            b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n')
+        self.write_line(
+            channel, b'MAIL from: <foo@example.com> size=20 body=8bitmime')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+    def test_with_enable_smtputf8_true(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), enable_SMTPUTF8=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(
+            channel,
+            b'MAIL from: <foo@example.com> size=20 body=8bitmime smtputf8')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+
 class SMTPDChannelTest(unittest.TestCase):
     def setUp(self):
         smtpd.socket = asyncore.socket = mock_socket
diff --git a/Misc/NEWS b/Misc/NEWS
index b18c5d7d74..cc2ff1dfde 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -38,6 +38,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #21795: smtpd now supports the 8BITMIME extension whenever
+  the new *decode_data* constructor argument is set to False.
+
 - Issue #21800: imaplib now supports RFC 5161 (enable), RFC 6855
   (utf8/internationalized email) and automatically encodes non-ASCII
   usernames and passwords to UTF8.