diff options
| author | Pablo Galindo <Pablogsal@gmail.com> | 2020-01-06 15:59:09 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-01-06 15:59:09 +0000 |
| commit | 5ec91f78d59d9c39b984f284e00cd04b96ddb5db (patch) | |
| tree | 3584ef9005da8bc532c12665147c8313142ff0e6 | |
| parent | 075ebad369d94342624792a41d3055c907bfa436 (diff) | |
| download | cpython-git-5ec91f78d59d9c39b984f284e00cd04b96ddb5db.tar.gz | |
bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)
| -rw-r--r-- | Lib/test/test_repl.py | 36 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst | 2 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 2 |
3 files changed, 40 insertions, 0 deletions
diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 9efd459a6f..71f192f90d 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -58,5 +58,41 @@ class TestInteractiveInterpreter(unittest.TestCase): # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr. self.assertIn(p.returncode, (1, 120)) + @cpython_only + def test_multiline_string_parsing(self): + # bpo-39209: Multiline string tokens need to be handled in the tokenizer + # in two places: the interactive path and the non-interactive path. + user_input = '''\ + x = """<?xml version="1.0" encoding="iso-8859-1"?> + <test> + <Users> + <fun25> + <limits> + <total>0KiB</total> + <kbps>0</kbps> + <rps>1.3</rps> + <connections>0</connections> + </limits> + <usages> + <total>16738211KiB</total> + <kbps>237.15</kbps> + <rps>1.3</rps> + <connections>0</connections> + </usages> + <time_to_refresh>never</time_to_refresh> + <limit_exceeded_URL>none</limit_exceeded_URL> + </fun25> + </Users> + </test>""" + ''' + user_input = dedent(user_input) + user_input = user_input.encode() + p = spawn_repl() + with SuppressCrashReport(): + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst new file mode 100644 index 0000000000..c05b3f8dfa --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst @@ -0,0 +1,2 @@ +Correctly handle multi-line tokens in interactive mode. Patch by Pablo +Galindo. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f84093dae5..f73c32684c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok) size_t start = tok->start - tok->buf; size_t oldlen = tok->cur - tok->buf; size_t newlen = oldlen + strlen(newtok); + Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf; char *buf = tok->buf; buf = (char *)PyMem_REALLOC(buf, newlen+1); tok->lineno++; @@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok) } tok->buf = buf; tok->cur = tok->buf + oldlen; + tok->multi_line_start = tok->buf + cur_multi_line_start; tok->line_start = tok->cur; strcpy(tok->buf + oldlen, newtok); PyMem_FREE(newtok); |