[2.7] bpo-23927: Make getargs.c skipitem() skipping 'w*'. (GH-8192). (GH-8255)

(cherry picked from commit 504373c59b48f1ea12132d515459022730db6047)

Also backport tests for skipitem() and handling errors.

4 files changed, 194 insertions, 2 deletions

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 944e9603cc..d281be6164 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -2,6 +2,7 @@
 # these are all functions _testcapi exports whose name begins with 'test_'.
 
 from __future__ import with_statement
+import string
 import sys
 import time
 import random
@@ -101,6 +102,133 @@ class TestPendingCalls(unittest.TestCase):
         self.pendingcalls_wait(l, n)
 
 
+# Bug #6012
+class Test6012(unittest.TestCase):
+    def test(self):
+        self.assertEqual(_testcapi.argparsing("Hello", "World"), 1)
+
+
+class SkipitemTest(unittest.TestCase):
+
+    def test_skipitem(self):
+        """
+        If this test failed, you probably added a new "format unit"
+        in Python/getargs.c, but neglected to update our poor friend
+        skipitem() in the same file.  (If so, shame on you!)
+
+        With a few exceptions**, this function brute-force tests all
+        printable ASCII*** characters (32 to 126 inclusive) as format units,
+        checking to see that PyArg_ParseTupleAndKeywords() return consistent
+        errors both when the unit is attempted to be used and when it is
+        skipped.  If the format unit doesn't exist, we'll get one of two
+        specific error messages (one for used, one for skipped); if it does
+        exist we *won't* get that error--we'll get either no error or some
+        other error.  If we get the specific "does not exist" error for one
+        test and not for the other, there's a mismatch, and the test fails.
+
+           ** Some format units have special funny semantics and it would
+              be difficult to accommodate them here.  Since these are all
+              well-established and properly skipped in skipitem() we can
+              get away with not testing them--this test is really intended
+              to catch *new* format units.
+
+          *** Python C source files must be ASCII.  Therefore it's impossible
+              to have non-ASCII format units.
+
+        """
+        empty_tuple = ()
+        tuple_1 = (0,)
+        dict_b = {'b':1}
+        keywords = ["a", "b"]
+
+        for i in range(32, 127):
+            c = chr(i)
+
+            # skip parentheses, the error reporting is inconsistent about them
+            # skip 'e', it's always a two-character code
+            # skip '|', it doesn't represent arguments anyway
+            if c in '()e|':
+                continue
+
+            # test the format unit when not skipped
+            format = c + "i"
+            try:
+                _testcapi.parse_tuple_and_keywords(tuple_1, dict_b,
+                    format, keywords)
+                when_not_skipped = False
+            except TypeError as e:
+                s = "argument 1 (impossible<bad format char>)"
+                when_not_skipped = (str(e) == s)
+            except RuntimeError:
+                when_not_skipped = False
+
+            # test the format unit when skipped
+            optional_format = "|" + format
+            try:
+                _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b,
+                    optional_format, keywords)
+                when_skipped = False
+            except RuntimeError as e:
+                s = "impossible<bad format char>: '{}'".format(format)
+                when_skipped = (str(e) == s)
+
+            message = ("test_skipitem_parity: "
+                "detected mismatch between convertsimple and skipitem "
+                "for format unit '{}' ({}), not skipped {}, skipped {}".format(
+                    c, i, when_skipped, when_not_skipped))
+            self.assertIs(when_skipped, when_not_skipped, message)
+
+    def test_skipitem_with_suffix(self):
+        parse = _testcapi.parse_tuple_and_keywords
+        empty_tuple = ()
+        tuple_1 = (0,)
+        dict_b = {'b':1}
+        keywords = ["a", "b"]
+
+        supported = ('s#', 's*', 'z#', 'z*', 'u#', 't#', 'w#', 'w*')
+        for c in string.ascii_letters:
+            for c2 in '#*':
+                f = c + c2
+                optional_format = "|" + f + "i"
+                if f in supported:
+                    parse(empty_tuple, dict_b, optional_format, keywords)
+                else:
+                    with self.assertRaisesRegexp((RuntimeError, TypeError),
+                                'impossible<bad format char>'):
+                        parse(empty_tuple, dict_b, optional_format, keywords)
+
+        for c in map(chr, range(32, 128)):
+            f = 'e' + c
+            optional_format = "|" + f + "i"
+            if c in 'st':
+                parse(empty_tuple, dict_b, optional_format, keywords)
+            else:
+                with self.assertRaisesRegexp(RuntimeError,
+                            'impossible<bad format char>'):
+                    parse(empty_tuple, dict_b, optional_format, keywords)
+
+    def test_parse_tuple_and_keywords(self):
+        # Test handling errors in the parse_tuple_and_keywords helper itself
+        self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, 42, [])
+        self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, '', 42)
+        self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, '', [''] * 42)
+        self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, '', [42])
+
+    def test_bad_use(self):
+        # Test handling invalid format and keywords in
+        # PyArg_ParseTupleAndKeywords()
+        self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords,
+                          (1,), {}, '||O', ['a'])
+        self.assertRaises(RuntimeError, _testcapi.parse_tuple_and_keywords,
+                          (1,), {}, '|O', ['a', 'b'])
+        self.assertRaises(RuntimeError, _testcapi.parse_tuple_and_keywords,
+                          (1,), {}, '|OO', ['a'])
+
+
 @unittest.skipUnless(threading and thread, 'Threading required for this test.')
 class TestThreadState(unittest.TestCase):
 
@@ -137,7 +265,8 @@ def test_main():
             except _testcapi.error:
                 raise support.TestFailed, sys.exc_info()[1]
 
-    support.run_unittest(CAPITest, TestPendingCalls, TestThreadState)
+    support.run_unittest(CAPITest, TestPendingCalls, SkipitemTest,
+                         TestThreadState)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Misc/NEWS.d/next/C API/2018-07-09-11-39-54.bpo-23927.pDFkxb.rst b/Misc/NEWS.d/next/C API/2018-07-09-11-39-54.bpo-23927.pDFkxb.rst
new file mode 100644
index 0000000000..3e2ac6c918
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2018-07-09-11-39-54.bpo-23927.pDFkxb.rst
@@ -0,0 +1,2 @@
+Fixed :exc:`SystemError` in :c:func:`PyArg_ParseTupleAndKeywords` when the
+``w*`` format unit is used for optional parameter.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index d87acc171c..f57176d632 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -1559,6 +1559,66 @@ getargs_et_hash(PyObject *self, PyObject *args)
     return result;
 }
 
+static PyObject *
+parse_tuple_and_keywords(PyObject *self, PyObject *args)
+{
+    PyObject *sub_args;
+    PyObject *sub_kwargs;
+    const char *sub_format;
+    PyObject *sub_keywords;
+
+    Py_ssize_t i, size;
+    char *keywords[8 + 1]; /* space for NULL at end */
+    PyObject *o;
+
+    int result;
+    PyObject *return_value = NULL;
+
+    double buffers[8][4]; /* double ensures alignment where necessary */
+
+    if (!PyArg_ParseTuple(args, "OOsO:parse_tuple_and_keywords",
+        &sub_args, &sub_kwargs,
+        &sub_format, &sub_keywords))
+        return NULL;
+
+    if (!(PyList_CheckExact(sub_keywords) || PyTuple_CheckExact(sub_keywords))) {
+        PyErr_SetString(PyExc_ValueError,
+            "parse_tuple_and_keywords: sub_keywords must be either list or tuple");
+        return NULL;
+    }
+
+    memset(buffers, 0, sizeof(buffers));
+    memset(keywords, 0, sizeof(keywords));
+
+    size = PySequence_Fast_GET_SIZE(sub_keywords);
+    if (size > 8) {
+        PyErr_SetString(PyExc_ValueError,
+            "parse_tuple_and_keywords: too many keywords in sub_keywords");
+        goto exit;
+    }
+
+    for (i = 0; i < size; i++) {
+        o = PySequence_Fast_GET_ITEM(sub_keywords, i);
+        keywords[i] = PyString_AsString(o);
+        if (keywords[i] == NULL) {
+            goto exit;
+        }
+    }
+
+    result = PyArg_ParseTupleAndKeywords(sub_args, sub_kwargs,
+        sub_format, keywords,
+        buffers + 0, buffers + 1, buffers + 2, buffers + 3,
+        buffers + 4, buffers + 5, buffers + 6, buffers + 7);
+
+    if (result) {
+        return_value = Py_None;
+        Py_INCREF(Py_None);
+    }
+
+exit:
+    return return_value;
+}
+
 #ifdef Py_USING_UNICODE
 
 static volatile int x;
@@ -2604,6 +2664,7 @@ static PyMethodDef TestMethods[] = {
 #ifdef Py_USING_UNICODE
     {"test_empty_argparse", (PyCFunction)test_empty_argparse,METH_NOARGS},
 #endif
+    {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS},
     {"test_null_strings",       (PyCFunction)test_null_strings,  METH_NOARGS},
     {"test_string_from_format", (PyCFunction)test_string_from_format, METH_NOARGS},
     {"test_with_docstring", (PyCFunction)test_with_docstring, METH_NOARGS,
diff --git a/Python/getargs.c b/Python/getargs.c
index 32ff6fc069..cc1ddde977 100644
--- a/Python/getargs.c
+++ b/Python/getargs.c
@@ -1780,7 +1780,7 @@ skipitem(const char **p_format, va_list *p_va, int flags)
                 else
                     (void) va_arg(*p_va, int *);
                 format++;
-            } else if ((c == 's' || c == 'z') && *format == '*') {
+            } else if ((c == 's' || c == 'z' || c == 'w') && *format == '*') {
                 format++;
             }
             break;