Make pgen compile with pydebug. Duplicate normalized names, as it may

be longer than the old string.

2 files changed, 8 insertions, 2 deletions

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 0ae093e68e..53c883f5e4 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -128,8 +128,10 @@ tok_new(void)
 	tok->read_coding_spec = 0;
 	tok->issued_encoding_warning = 0;
 	tok->encoding = NULL;
+#ifndef PGEN
 	tok->decoding_readline = NULL;
 	tok->decoding_buffer = NULL;
+#endif
 	return tok;
 }
 
@@ -225,8 +227,8 @@ get_coding_spec(const char *s, int size)
 				char* r = new_string(begin, t - begin);
 				char* q = get_normal_name(r);
 				if (r != q) {
-					assert(strlen(r) >= strlen(q));
-					strcpy(r, q);
+					PyMem_DEL(r);
+					r = new_string(q, strlen(q));
 				}
 				return r;
 			}
@@ -584,8 +586,10 @@ PyTokenizer_Free(struct tok_state *tok)
 {
 	if (tok->encoding != NULL)
 		PyMem_DEL(tok->encoding);
+#ifndef PGEN
 	Py_XDECREF(tok->decoding_readline);
 	Py_XDECREF(tok->decoding_buffer);
+#endif
 	if (tok->fp != NULL && tok->buf != NULL)
 		PyMem_DEL(tok->buf);
 	PyMem_DEL(tok);
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 9782666fd8..f3bac74acd 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -45,8 +45,10 @@ struct tok_state {
 	int read_coding_spec;	/* whether 'coding:...' has been read  */
 	int issued_encoding_warning; /* whether non-ASCII warning was issued */
 	char *encoding;
+#ifndef PGEN
 	PyObject *decoding_readline; /* codecs.open(...).readline */
 	PyObject *decoding_buffer;
+#endif
 	const char* enc;
 	const char* str;
 };