Issue #20390: Small fixes and improvements for Argument Clinic.

18 files changed, 164 insertions, 125 deletions

diff --git a/Include/object.h b/Include/object.h
index a09421bd42..015d216993 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -492,8 +492,12 @@ PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *
 PyAPI_FUNC(unsigned int) PyType_ClearCache(void);
 PyAPI_FUNC(void) PyType_Modified(PyTypeObject *);
 
-PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *name, const char *internal_doc);
-PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_doc);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *)
+_PyType_GetDocFromInternalDoc(const char *, const char *);
+PyAPI_FUNC(PyObject *)
+_PyType_GetTextSignatureFromInternalDoc(const char *, const char *);
+#endif
 
 /* Generic operations on objects */
 struct _Py_Identifier;
diff --git a/Misc/NEWS b/Misc/NEWS
index 918739135f..f9ff1bef29 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -2,10 +2,10 @@
 Python News
 +++++++++++
 
-What's New in Python 3.4.0 Release Candidate 1?
-===============================================
+What's New in Python 3.4.0 Beta 3?
+==================================
 
-Release date: 2014-01-19
+Release date: 2014-01-25
 
 Core and Builtins
 -----------------
@@ -147,6 +147,23 @@ Tests
 Tools/Demos
 -----------
 
+- Issue #20390: Argument Clinic's "file" output preset now defaults to
+  "{dirname}/clinic/{basename}.h".
+
+- Issue #20390: Argument Clinic's "class" directive syntax has been extended
+  with two new required  arguments: "typedef" and "type_object".
+
+- Issue #20390: Argument Clinic: If __new__ or __init__ functions didn't use
+  kwargs (or args), the PyArg_NoKeywords (or PyArg_NoPositional) calls
+  generated are only run when the type object is an exact match.
+
+- Issue #20390: Argument Clinic now fails if you have required parameters after
+  optional parameters.
+
+- Issue #20390: Argument Clinic converters now have a new template they can
+  inject code into: "modifiers".  Code put there is run in the parsing
+  function after argument parsing but before the call to the impl.
+
 - Issue #20376: Argument Clinic now escapes backslashes in docstrings.
 
 - Issue #20381: Argument Clinic now sanity checks the default argument when
diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c
index cdaa837ff0..30e034046c 100644
--- a/Modules/_bz2module.c
+++ b/Modules/_bz2module.c
@@ -199,8 +199,8 @@ error:
 /*[clinic input]
 output preset file
 module _bz2
-class _bz2.BZ2Compressor
-class _bz2.BZ2Decompressor
+class _bz2.BZ2Compressor "BZ2Compressor *" "&BZ2Compressor_Type"
+class _bz2.BZ2Decompressor "BZ2Decompressor *" "&BZ2Decompressor_Type"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
@@ -209,7 +209,6 @@ class _bz2.BZ2Decompressor
 /*[clinic input]
 _bz2.BZ2Compressor.compress
 
-    self: self(type="BZ2Compressor *")
     data: Py_buffer
     /
 
@@ -239,8 +238,6 @@ _bz2_BZ2Compressor_compress_impl(BZ2Compressor *self, Py_buffer *data)
 /*[clinic input]
 _bz2.BZ2Compressor.flush
 
-    self: self(type="BZ2Compressor *")
-
 Finish the compression process.
 
 Returns the compressed data left in internal buffers.
@@ -294,7 +291,6 @@ BZ2_Free(void* ctx, void *ptr)
 /*[clinic input]
 _bz2.BZ2Compressor.__init__
 
-    self: self(type="BZ2Compressor *")
     compresslevel: int = 9
         Compression level, as a number between 1 and 9.
     /
@@ -472,7 +468,6 @@ error:
 /*[clinic input]
 _bz2.BZ2Decompressor.decompress
 
-    self: self(type="BZ2Decompressor *")
     data: Py_buffer
     /
 
@@ -511,8 +506,6 @@ BZ2Decompressor_getstate(BZ2Decompressor *self, PyObject *noargs)
 /*[clinic input]
 _bz2.BZ2Decompressor.__init__
 
-    self: self(type="BZ2Decompressor *")
-
 Create a decompressor object for decompressing data incrementally.
 
 For one-shot decompression, use the decompress() function instead.
diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c
index d1b30198ff..534dcff681 100644
--- a/Modules/_cursesmodule.c
+++ b/Modules/_cursesmodule.c
@@ -136,7 +136,7 @@ typedef chtype attr_t;           /* No attr_t type is available */
 
 /*[clinic input]
 module curses
-class curses.window
+class curses.window "PyCursesWindowObject *" "&PyCursesWindow_Type"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
@@ -605,10 +605,10 @@ PyDoc_STRVAR(curses_window_addch__doc__,
     {"addch", (PyCFunction)curses_window_addch, METH_VARARGS, curses_window_addch__doc__},
 
 static PyObject *
-curses_window_addch_impl(PyObject *self, int group_left_1, int x, int y, PyObject *ch, int group_right_1, long attr);
+curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, int x, int y, PyObject *ch, int group_right_1, long attr);
 
 static PyObject *
-curses_window_addch(PyObject *self, PyObject *args)
+curses_window_addch(PyCursesWindowObject *self, PyObject *args)
 {
     PyObject *return_value = NULL;
     int group_left_1 = 0;
@@ -650,8 +650,8 @@ exit:
 }
 
 static PyObject *
-curses_window_addch_impl(PyObject *self, int group_left_1, int x, int y, PyObject *ch, int group_right_1, long attr)
-/*[clinic end generated code: checksum=f6eeada77a9ec085125f3a27e4a2095f2a4c50be]*/
+curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, int x, int y, PyObject *ch, int group_right_1, long attr)
+/*[clinic end generated code: checksum=e1cdbd4f4e42fc6b36fd4755d7e4bd5b58751ea1]*/
 {
     PyCursesWindowObject *cwself = (PyCursesWindowObject *)self;
     int coordinates_group = group_left_1;
diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index 4dfb84cc48..b78725e3bb 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -18,7 +18,7 @@
 
 /*[clinic input]
 module datetime
-class datetime.datetime
+class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c
index 6cf23287c9..88dae16560 100644
--- a/Modules/_dbmmodule.c
+++ b/Modules/_dbmmodule.c
@@ -30,7 +30,7 @@ static char *which_dbm = "Berkeley DB";
 
 /*[clinic input]
 module dbm
-class dbm.dbm
+class dbm.dbm "dbmobject *" "&Dbmtype"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index fa5f73df10..dabd81e3ab 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -6,28 +6,13 @@ PyDoc_STRVAR(pickle_module_doc,
 
 /*[clinic input]
 module _pickle
-class _pickle.Pickler
-class _pickle.PicklerMemoProxy
-class _pickle.Unpickler
-class _pickle.UnpicklerMemoProxy
+class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
+class _pickle.PicklerMemoProxy "PicklerMemoProxyObject *" "&PicklerMemoProxyType"
+class _pickle.Unpickler "UnpicklerObject *" "&Unpickler_Type"
+class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "&UnpicklerMemoProxyType"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
-/*[python input]
-class PicklerObject_converter(self_converter):
-    type = "PicklerObject *"
-
-class PicklerMemoProxyObject_converter(self_converter):
-    type = "PicklerMemoProxyObject *"
-
-class UnpicklerObject_converter(self_converter):
-    type = "UnpicklerObject *"
-
-class UnpicklerMemoProxyObject_converter(self_converter):
-    type = "UnpicklerMemoProxyObject *"
-[python start generated code]*/
-/*[python end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
-
 /* Bump this when new opcodes are added to the pickle protocol. */
 enum {
     HIGHEST_PROTOCOL = 4,
@@ -3878,8 +3863,6 @@ dump(PicklerObject *self, PyObject *obj)
 
 _pickle.Pickler.clear_memo
 
-  self: PicklerObject
-
 Clears the pickler's "memo".
 
 The memo is the data structure that remembers which objects the
@@ -3923,7 +3906,6 @@ _pickle_Pickler_clear_memo_impl(PicklerObject *self)
 
 _pickle.Pickler.dump
 
-  self: PicklerObject
   obj: object
   /
 
@@ -4018,7 +4000,6 @@ Pickler_clear(PicklerObject *self)
 
 _pickle.Pickler.__init__
 
-  self: PicklerObject
   file: object
   protocol: object = NULL
   fix_imports: bool = True
@@ -4158,8 +4139,6 @@ typedef struct {
 /*[clinic input]
 _pickle.PicklerMemoProxy.clear
 
-  self: PicklerMemoProxyObject
-
 Remove all items from memo.
 [clinic start generated code]*/
 
@@ -4191,8 +4170,6 @@ _pickle_PicklerMemoProxy_clear_impl(PicklerMemoProxyObject *self)
 /*[clinic input]
 _pickle.PicklerMemoProxy.copy
 
-  self: PicklerMemoProxyObject
-
 Copy the memo to a new object.
 [clinic start generated code]*/
 
@@ -4254,8 +4231,6 @@ _pickle_PicklerMemoProxy_copy_impl(PicklerMemoProxyObject *self)
 /*[clinic input]
 _pickle.PicklerMemoProxy.__reduce__
 
-  self: PicklerMemoProxyObject
-
 Implement pickle support.
 [clinic start generated code]*/
 
@@ -6310,17 +6285,17 @@ PyDoc_STRVAR(_pickle_Unpickler_load__doc__,
     {"load", (PyCFunction)_pickle_Unpickler_load, METH_NOARGS, _pickle_Unpickler_load__doc__},
 
 static PyObject *
-_pickle_Unpickler_load_impl(PyObject *self);
+_pickle_Unpickler_load_impl(UnpicklerObject *self);
 
 static PyObject *
-_pickle_Unpickler_load(PyObject *self, PyObject *Py_UNUSED(ignored))
+_pickle_Unpickler_load(UnpicklerObject *self, PyObject *Py_UNUSED(ignored))
 {
     return _pickle_Unpickler_load_impl(self);
 }
 
 static PyObject *
-_pickle_Unpickler_load_impl(PyObject *self)
-/*[clinic end generated code: checksum=fb1119422c5e03045d690d1cd6c457f1ca4c585d]*/
+_pickle_Unpickler_load_impl(UnpicklerObject *self)
+/*[clinic end generated code: checksum=5ccece694e9898856d916e0a87f0133d4537ebb9]*/
 {
     UnpicklerObject *unpickler = (UnpicklerObject*)self;
 
@@ -6347,7 +6322,6 @@ _pickle_Unpickler_load_impl(PyObject *self)
 
 _pickle.Unpickler.find_class
 
-  self: UnpicklerObject
   module_name: object
   global_name: object
   /
@@ -6551,7 +6525,6 @@ Unpickler_clear(UnpicklerObject *self)
 
 _pickle.Unpickler.__init__
 
-  self: UnpicklerObject
   file: object
   *
   fix_imports: bool = True
@@ -6692,8 +6665,6 @@ typedef struct {
 /*[clinic input]
 _pickle.UnpicklerMemoProxy.clear
 
-  self: UnpicklerMemoProxyObject
-
 Remove all items from memo.
 [clinic start generated code]*/
 
@@ -6727,8 +6698,6 @@ _pickle_UnpicklerMemoProxy_clear_impl(UnpicklerMemoProxyObject *self)
 /*[clinic input]
 _pickle.UnpicklerMemoProxy.copy
 
-  self: UnpicklerMemoProxyObject
-
 Copy the memo to a new object.
 [clinic start generated code]*/
 
@@ -6783,8 +6752,6 @@ error:
 /*[clinic input]
 _pickle.UnpicklerMemoProxy.__reduce__
 
-  self: UnpicklerMemoProxyObject
-
 Implement pickling support.
 [clinic start generated code]*/
 
diff --git a/Modules/_sre.c b/Modules/_sre.c
index 776d6eb2dd..41dca7d97d 100644
--- a/Modules/_sre.c
+++ b/Modules/_sre.c
@@ -528,11 +528,10 @@ sre_search(SRE_STATE* state, SRE_CODE* pattern)
 
 /*[clinic input]
 module _sre
-class _sre.SRE_Pattern
+class _sre.SRE_Pattern "PatternObject *" "&Pattern_Type"
 
 _sre.SRE_Pattern.match as pattern_match
 
-    self: self(type="PatternObject *")
     pattern: object
     pos: Py_ssize_t = 0
     endpos: Py_ssize_t(c_default="PY_SSIZE_T_MAX") = sys.maxsize
diff --git a/Modules/audioop.c b/Modules/audioop.c
index 7e0aef58a8..d9a22c5a26 100644
--- a/Modules/audioop.c
+++ b/Modules/audioop.c
@@ -393,7 +393,6 @@ audioop_check_parameters(Py_ssize_t len, int size)
 /*[clinic input]
 output preset file
 module audioop
-class audioop.error
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
diff --git a/Modules/_bz2module.clinic.c b/Modules/clinic/_bz2module.c.h
index 68250af86c..1ba87cfb5c 100644
--- a/Modules/_bz2module.clinic.c
+++ b/Modules/clinic/_bz2module.c.h
@@ -75,7 +75,8 @@ _bz2_BZ2Compressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
     int return_value = -1;
     int compresslevel = 9;
 
-    if (!_PyArg_NoKeywords("BZ2Compressor", kwargs))
+    if (({self_name} == {self_type_object}) &&
+        !_PyArg_NoKeywords("BZ2Compressor", kwargs))
         goto exit;
     if (!PyArg_ParseTuple(args,
         "|i:BZ2Compressor",
@@ -137,13 +138,15 @@ _bz2_BZ2Decompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
 {
     int return_value = -1;
 
-    if (!_PyArg_NoPositional("BZ2Decompressor", args))
+    if (({self_name} == {self_type_object}) &&
+        !_PyArg_NoPositional("BZ2Decompressor", args))
         goto exit;
-    if (!_PyArg_NoKeywords("BZ2Decompressor", kwargs))
+    if (({self_name} == {self_type_object}) &&
+        !_PyArg_NoKeywords("BZ2Decompressor", kwargs))
         goto exit;
     return_value = _bz2_BZ2Decompressor___init___impl((BZ2Decompressor *)self);
 
 exit:
     return return_value;
 }
-/*[clinic end generated code: checksum=9bb33ae7d35494b7a5365f03f390e4b5b8b1bc49]*/
+/*[clinic end generated code: checksum=79ee0d9731dfe404baec35b704b2ca2179b9a6c0]*/
diff --git a/Modules/_lzmamodule.c b/Modules/clinic/_lzmamodule.c.h
index 4e1889278c..c8899c9abf 100644
--- a/Modules/_lzmamodule.c
+++ b/Modules/clinic/_lzmamodule.c.h
@@ -472,8 +472,8 @@ error:
 /*[clinic input]
 output preset file
 module _lzma
-class _lzma.LZMACompressor
-class _lzma.LZMADecompressor
+class _lzma.LZMACompressor "Compressor *" "&Compressor_type"
+class _lzma.LZMADecompressor "Decompressor *" "&Decompressor_type"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
diff --git a/Modules/audioop.clinic.c b/Modules/clinic/audioop.c.h
index 974fdb1df5..974fdb1df5 100644
--- a/Modules/audioop.clinic.c
+++ b/Modules/clinic/audioop.c.h
diff --git a/Modules/binascii.clinic.c b/Modules/clinic/binascii.c.h
index 3e4a468967..3e4a468967 100644
--- a/Modules/binascii.clinic.c
+++ b/Modules/clinic/binascii.c.h
diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c
index ff13e0b3bb..ca1620feab 100644
--- a/Modules/unicodedata.c
+++ b/Modules/unicodedata.c
@@ -19,7 +19,7 @@
 
 /*[clinic input]
 module unicodedata
-class unicodedata.UCD
+class unicodedata.UCD 'PreviousDBVersion *' '&UCD_Type'
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
@@ -140,10 +140,10 @@ PyDoc_STRVAR(unicodedata_UCD_decimal__doc__,
     {"decimal", (PyCFunction)unicodedata_UCD_decimal, METH_VARARGS, unicodedata_UCD_decimal__doc__},
 
 static PyObject *
-unicodedata_UCD_decimal_impl(PyObject *self, PyUnicodeObject *unichr, PyObject *default_value);
+unicodedata_UCD_decimal_impl(PreviousDBVersion *self, PyUnicodeObject *unichr, PyObject *default_value);
 
 static PyObject *
-unicodedata_UCD_decimal(PyObject *self, PyObject *args)
+unicodedata_UCD_decimal(PreviousDBVersion *self, PyObject *args)
 {
     PyObject *return_value = NULL;
     PyUnicodeObject *unichr;
@@ -160,15 +160,14 @@ exit:
 }
 
 static PyObject *
-unicodedata_UCD_decimal_impl(PyObject *self, PyUnicodeObject *unichr, PyObject *default_value)
-/*[clinic end generated code: checksum=01826b179d497d8fd3842c56679ecbd4faddaa95]*/
+unicodedata_UCD_decimal_impl(PreviousDBVersion *self, PyUnicodeObject *unichr, PyObject *default_value)
+/*[clinic end generated code: checksum=e1371a1a016e19fdd3cd2c1af1d1832df095f50b]*/
 {
-    PyUnicodeObject *v = (PyUnicodeObject *)unichr;
     int have_old = 0;
     long rc;
     Py_UCS4 c;
 
-    c = getuchar(v);
+    c = getuchar(unichr);
     if (c == (Py_UCS4)-1)
         return NULL;
 
diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
index 24f1e3966f..78ee10b5a6 100644
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -83,8 +83,8 @@ zlib_error(z_stream zst, int err, char *msg)
 
 /*[clinic input]
 module zlib
-class zlib.Compress
-class zlib.Decompress
+class zlib.Compress "compobject *" "&Comptype"
+class zlib.Decompress "compobject *" "&Decomptype"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
@@ -748,8 +748,6 @@ save_unconsumed_input(compobject *self, int err)
 
 zlib.Decompress.decompress
 
-    self: self(type="compobject *")
-
     data: Py_buffer
         The binary data to decompress.
     max_length: uint = 0
@@ -1030,8 +1028,6 @@ PyZlib_flush(compobject *self, PyObject *args)
 /*[clinic input]
 zlib.Compress.copy
 
-    self: self(type="compobject *")
-
 Return a copy of the compression object.
 [clinic start generated code]*/
 
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 8eff9990cd..2332e3fcd8 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -70,7 +70,7 @@ to the combined-table form.
 #include "stringlib/eq.h"
 
 /*[clinic input]
-class dict
+class dict "PyDictObject *" "&PyDict_Type"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
@@ -1694,7 +1694,6 @@ dict_items(PyDictObject *mp)
 /*[clinic input]
 @classmethod
 dict.fromkeys
-
     iterable: object
     value: object=None
     /
@@ -2217,10 +2216,10 @@ PyDoc_STRVAR(dict___contains____doc__,
     {"__contains__", (PyCFunction)dict___contains__, METH_O|METH_COEXIST, dict___contains____doc__},
 
 static PyObject *
-dict___contains__(PyObject *self, PyObject *key)
-/*[clinic end generated code: checksum=c4f85a39baac4776c4275ad5f072f7732c5f0806]*/
+dict___contains__(PyDictObject *self, PyObject *key)
+/*[clinic end generated code: checksum=744ca54369dda9815a596304087f1b37fafa5960]*/
 {
-    register PyDictObject *mp = (PyDictObject *)self;
+    register PyDictObject *mp = self;
     Py_hash_t hash;
     PyDictKeyEntry *ep;
     PyObject **value_addr;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 9acd37fe13..919b7333d7 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -48,7 +48,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #endif
 
 /*[clinic input]
-class str
+class str "PyUnicodeObject *" "&PyUnicode_Type"
 [clinic start generated code]*/
 /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
 
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index 7f1b059e80..0dec2589d1 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -300,6 +300,10 @@ class CRenderData:
         # Should be full lines with \n eol characters.
         self.initializers = []
 
+        # The C statements needed to dynamically modify the values
+        # parsed by the parse call, before calling the impl.
+        self.modifications = []
+
         # The entries for the "keywords" array for PyArg_ParseTuple.
         # Should be individual strings representing the names.
         self.keywords = []
@@ -541,6 +545,7 @@ __________________________________________________
 
 parser_definition_impl_call
 
+    {modifications}
     {return_value} = {c_basename}_impl({impl_arguments});
 __________________________________________________
 
@@ -575,14 +580,14 @@ __________________________________________________
 
 parser_definition_no_positional
 
-    if (!_PyArg_NoPositional("{name}", args))
+    if ({self_type_check}!_PyArg_NoPositional("{name}", args))
         goto exit;
 
 __________________________________________________
 
 parser_definition_no_keywords
 
-    if (!_PyArg_NoKeywords("{name}", kwargs))
+    if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs))
         goto exit;
 
 __________________________________________________
@@ -691,7 +696,7 @@ __________________________________________________
         impl_definition = templates['impl_definition']
         impl_prototype = parser_prototype = parser_definition = None
 
-        parser_body_fields = None
+        parser_body_fields = ()
         def parser_body(prototype, *fields):
             nonlocal parser_body_fields
             add, output = text_accumulator()
@@ -1025,6 +1030,7 @@ __________________________________________________
 
         template_dict['docstring'] = self.docstring_for_c_string(f)
 
+        template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = ''
         f_self.converter.set_template_dict(template_dict)
 
         f.return_converter.render(f, data)
@@ -1032,6 +1038,7 @@ __________________________________________________
 
         template_dict['declarations'] = "\n".join(data.declarations)
         template_dict['initializers'] = "\n\n".join(data.initializers)
+        template_dict['modifications'] = '\n\n'.join(data.modifications)
         template_dict['keywords'] = '"' + '", "'.join(data.keywords) + '"'
         template_dict['format_units'] = ''.join(data.format_units)
         template_dict['parse_arguments'] = ', '.join(data.parse_arguments)
@@ -1060,6 +1067,7 @@ __________________________________________________
                 declarations=template_dict['declarations'],
                 return_conversion=template_dict['return_conversion'],
                 initializers=template_dict['initializers'],
+                modifications=template_dict['modifications'],
                 cleanup=template_dict['cleanup'],
                 )
 
@@ -1356,8 +1364,14 @@ class Destination:
             fail("Too many arguments for destination " + name + " new " + type)
         if type =='file':
             d = {}
-            d['filename'] = filename = clinic.filename
-            d['basename'], d['extension'] = os.path.splitext(filename)
+            filename = clinic.filename
+            d['path'] = filename
+            dirname, basename = os.path.split(filename)
+            if not dirname:
+                dirname = '.'
+            d['dirname'] = dirname
+            d['basename'] = basename
+            d['basename_root'], d['basename_extension'] = os.path.splitext(filename)
             self.filename = args[0].format_map(d)
         if type == 'two-pass':
             self.id = None
@@ -1476,7 +1490,7 @@ impl_definition block
         self.add_destination("buffer", "buffer")
         self.add_destination("two-pass", "two-pass")
         if filename:
-            self.add_destination("file", "file", "{basename}.clinic{extension}")
+            self.add_destination("file", "file", "{dirname}/clinic/{basename}.h")
 
         d = self.destinations.get
         self.field_destinations = collections.OrderedDict((
@@ -1572,6 +1586,14 @@ impl_definition block
 
                 if destination.type == 'file':
                     try:
+                        dirname = os.path.dirname(destination.filename)
+                        try:
+                            os.makedirs(dirname)
+                        except FileExistsError:
+                            if not os.path.isdir(dirname):
+                                fail("Can't write to destination {}, "
+                                     "can't make directory {}!".format(
+                                        destination.filename, dirname))
                         with open(destination.filename, "rt") as f:
                             parser_2 = BlockParser(f.read(), language=self.language)
                             blocks = list(parser_2)
@@ -1696,10 +1718,12 @@ class Module:
         return "<clinic.Module " + repr(self.name) + " at " + str(id(self)) + ">"
 
 class Class:
-    def __init__(self, name, module=None, cls=None):
+    def __init__(self, name, module=None, cls=None, typedef=None, type_object=None):
         self.name = name
         self.module = module
         self.cls = cls
+        self.typedef = typedef
+        self.type_object = type_object
         self.parent = cls or module
 
         self.classes = collections.OrderedDict()
@@ -1980,6 +2004,7 @@ class CConverter(metaclass=CConverterAutoRegister):
 
     # Should we show this parameter in the generated
     # __text_signature__? This is *almost* always True.
+    # (It's only False for __new__, __init__, and METH_STATIC functions.)
     show_in_signature = True
 
     # Overrides the name used in a text signature.
@@ -2050,6 +2075,11 @@ class CConverter(metaclass=CConverterAutoRegister):
         if initializers:
             data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip())
 
+        # modifications
+        modifications = self.modify()
+        if modifications:
+            data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip())
+
         # keywords
         data.keywords.append(original_name)
 
@@ -2152,6 +2182,14 @@ class CConverter(metaclass=CConverterAutoRegister):
         """
         return ""
 
+    def modify(self):
+        """
+        The C statements required to modify this variable after parsing.
+        Returns a string containing this code indented at column 0.
+        If no initialization is necessary, returns an empty string.
+        """
+        return ""
+
     def cleanup(self):
         """
         The C statements required to clean up after this variable.
@@ -2463,6 +2501,12 @@ def correct_name_for_self(f):
         return "PyTypeObject *", "type"
     raise RuntimeError("Unhandled type of function f: " + repr(f.kind))
 
+def required_type_for_self_for_parser(f):
+    type, _ = correct_name_for_self(f)
+    if f.kind in (METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD):
+        return type
+    return None
+
 
 class self_converter(CConverter):
     """
@@ -2526,12 +2570,7 @@ class self_converter(CConverter):
 
     @property
     def parser_type(self):
-        kind = self.function.kind
-        if kind == METHOD_NEW:
-            return "PyTypeObject *"
-        if kind == METHOD_INIT:
-            return "PyObject *"
-        return self.type
+        return required_type_for_self_for_parser(self.function) or self.type
 
     def render(self, parameter, data):
         """
@@ -2554,6 +2593,7 @@ class self_converter(CConverter):
     def set_template_dict(self, template_dict):
         template_dict['self_name'] = self.name
         template_dict['self_type'] = self.parser_type
+        template_dict['self_type_check'] = '({self_name} == {self_type_object}) &&\n        '
 
 
 
@@ -2808,6 +2848,7 @@ class DSLParser:
         self.keyword_only = False
         self.group = 0
         self.parameter_state = self.ps_start
+        self.seen_positional_with_default = False
         self.indent = IndentStack()
         self.kind = CALLABLE
         self.coexist = False
@@ -2825,11 +2866,15 @@ class DSLParser:
         module, cls = self.clinic._module_and_class(fields)
         if cls:
             fail("Can't nest a module inside a class!")
+
+        if name in module.classes:
+            fail("Already defined module " + repr(name) + "!")
+
         m = Module(name, module)
         module.modules[name] = m
         self.block.signatures.append(m)
 
-    def directive_class(self, name):
+    def directive_class(self, name, typedef, type_object):
         fields = name.split('.')
         in_classes = False
         parent = self
@@ -2837,11 +2882,12 @@ class DSLParser:
         so_far = []
         module, cls = self.clinic._module_and_class(fields)
 
-        c = Class(name, module, cls)
-        if cls:
-            cls.classes[name] = c
-        else:
-            module.classes[name] = c
+        parent = cls or module
+        if name in parent.classes:
+            fail("Already defined class " + repr(name) + "!")
+
+        c = Class(name, module, cls, typedef, type_object)
+        parent.classes[name] = c
         self.block.signatures.append(c)
 
     def directive_set(self, name, value):
@@ -3035,6 +3081,8 @@ class DSLParser:
                 else:
                     existing_function = None
                 if not existing_function:
+                    print("class", cls, "module", module, "exsiting", existing)
+                    print("cls. functions", cls.functions)
                     fail("Couldn't find existing function " + repr(existing) + "!")
 
                 fields = [x.strip() for x in full_name.split('.')]
@@ -3113,8 +3161,11 @@ class DSLParser:
         self.block.signatures.append(self.function)
 
         # insert a self converter automatically
-        _, name = correct_name_for_self(self.function)
-        sc = self.function.self_converter = self_converter(name, self.function)
+        type, name = correct_name_for_self(self.function)
+        kwargs = {}
+        if cls and type == "PyObject *":
+            kwargs['type'] = cls.typedef
+        sc = self.function.self_converter = self_converter(name, self.function, **kwargs)
         p_self = Parameter(sc.name, inspect.Parameter.POSITIONAL_ONLY, function=self.function, converter=sc)
         self.function.parameters[sc.name] = p_self
 
@@ -3175,18 +3226,21 @@ class DSLParser:
     # "parameter_state".  (Previously the code was a miasma of ifs and
     # separate boolean state variables.)  The states are:
     #
-    #  [ [ a, b, ] c, ] d, e, f, [ g, h, [ i ] ] /   <- line
-    # 01   2          3          4           5   6   <- state transitions
+    #  [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] /   <- line
+    # 01   2          3       4    5           6   7   <- state transitions
     #
     # 0: ps_start.  before we've seen anything.  legal transitions are to 1 or 3.
     # 1: ps_left_square_before.  left square brackets before required parameters.
     # 2: ps_group_before.  in a group, before required parameters.
-    # 3: ps_required.  required parameters.  (renumber left groups!)
-    # 4: ps_group_after.  in a group, after required parameters.
-    # 5: ps_right_square_after.  right square brackets after required parameters.
-    # 6: ps_seen_slash.  seen slash.
+    # 3: ps_required.  required parameters, positional-or-keyword or positional-only
+    #     (we don't know yet).  (renumber left groups!)
+    # 4: ps_optional.  positional-or-keyword or positional-only parameters that
+    #    now must have default values.
+    # 5: ps_group_after.  in a group, after required parameters.
+    # 6: ps_right_square_after.  right square brackets after required parameters.
+    # 7: ps_seen_slash.  seen slash.
     ps_start, ps_left_square_before, ps_group_before, ps_required, \
-    ps_group_after, ps_right_square_after, ps_seen_slash = range(7)
+    ps_optional, ps_group_after, ps_right_square_after, ps_seen_slash = range(8)
 
     def state_parameters_start(self, line):
         if self.ignore_line(line):
@@ -3245,21 +3299,25 @@ class DSLParser:
         elif self.parameter_state == self.ps_group_before:
             if not self.group:
                 self.to_required()
-        elif self.parameter_state == self.ps_group_after:
+        elif self.parameter_state in (self.ps_group_after, self.ps_optional):
             pass
         else:
-            fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")")
+            fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".a)")
 
         base, equals, default = line.rpartition('=')
         if not equals:
             base = default
             default = None
+
         module = None
         try:
             ast_input = "def x({}): pass".format(base)
             module = ast.parse(ast_input)
         except SyntaxError:
             try:
+                # the last = was probably inside a function call, like
+                #   i: int(nullable=True)
+                # so assume there was no actual default value.
                 default = None
                 ast_input = "def x({}): pass".format(line)
                 module = ast.parse(ast_input)
@@ -3275,13 +3333,18 @@ class DSLParser:
         name, legacy, kwargs = self.parse_converter(parameter.annotation)
 
         if not default:
+            if self.parameter_state == self.ps_optional:
+                fail("Can't have a parameter without a default (" + repr(parameter_name) + ")\nafter a parameter with a default!")
             value = unspecified
             if 'py_default' in kwargs:
                 fail("You can't specify py_default without specifying a default value!")
         else:
+            if self.parameter_state == self.ps_required:
+                self.parameter_state = self.ps_optional
             default = default.strip()
             bad = False
             ast_input = "x = {}".format(default)
+            bad = False
             try:
                 module = ast.parse(ast_input)
 
@@ -3441,7 +3504,7 @@ class DSLParser:
             elif self.parameter_state in (self.ps_required, self.ps_group_after):
                 self.parameter_state = self.ps_group_after
             else:
-                fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")")
+                fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".b)")
             self.group += 1
         elif symbol == ']':
             if not self.group:
@@ -3454,12 +3517,12 @@ class DSLParser:
             elif self.parameter_state in (self.ps_group_after, self.ps_right_square_after):
                 self.parameter_state = self.ps_right_square_after
             else:
-                fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")")
+                fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".c)")
         elif symbol == '/':
-            # ps_required is allowed here, that allows positional-only without option groups
+            # ps_required and ps_optional are allowed here, that allows positional-only without option groups
             # to work (and have default values!)
-            if (self.parameter_state not in (self.ps_required, self.ps_right_square_after, self.ps_group_before)) or self.group:
-                fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")")
+            if (self.parameter_state not in (self.ps_required, self.ps_optional, self.ps_right_square_after, self.ps_group_before)) or self.group:
+                fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".d)")
             if self.keyword_only:
                 fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.")
             self.parameter_state = self.ps_seen_slash