bpo-40334: Rename PyConfig.use_peg to _use_peg_parser (GH-19670)

* Rename PyConfig.use_peg to _use_peg_parser
* Document PyConfig._use_peg_parser and mark it a deprecated
* Mark -X oldparser option and PYTHONOLDPARSER env var as deprecated
  in the documentation.
* Add use_old_parser() and skip_if_new_parser() to test.support
* Remove sys.flags.use_peg: use_old_parser() uses
  _testinternalcapi.get_configs() instead.
* Enhance test_embed tests
* subprocess._args_from_interpreter_flags() copies -X oldparser

18 files changed, 46 insertions, 32 deletions

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 86fdf27f9b..13600c28cf 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -326,7 +326,7 @@ def _args_from_interpreter_flags():
     if dev_mode:
         args.extend(('-X', 'dev'))
     for opt in ('faulthandler', 'tracemalloc', 'importtime',
-                'showrefcount', 'utf8'):
+                'showrefcount', 'utf8', 'oldparser'):
         if opt in xoptions:
             value = xoptions[opt]
             if value is True:
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 9f43b4071c..4fe247aeb9 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -3454,3 +3454,13 @@ def wait_process(pid, *, exitcode, timeout=None):
     # sanity check: it should not fail in practice
     if pid2 != pid:
         raise AssertionError(f"pid {pid2} != pid {pid}")
+
+
+def use_old_parser():
+    import _testinternalcapi
+    config = _testinternalcapi.get_configs()
+    return (config['config']['_use_peg_parser'] == 0)
+
+
+def skip_if_new_parser(msg):
+    return unittest.skipIf(not use_old_parser(), msg)
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
index f1d74b1fb7..8c3e447200 100644
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -4,12 +4,12 @@
 """
 import sys
 import unittest
-from test.support import is_jython
+from test import support
 
 from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 import io
 
-if is_jython:
+if support.is_jython:
 
     def unify_callables(d):
         for n,v in d.items():
@@ -21,7 +21,7 @@ class CodeopTests(unittest.TestCase):
 
     def assertValid(self, str, symbol='single'):
         '''succeed iff str is a valid piece of code'''
-        if is_jython:
+        if support.is_jython:
             code = compile_command(str, "<input>", symbol)
             self.assertTrue(code)
             if symbol == "single":
@@ -60,7 +60,7 @@ class CodeopTests(unittest.TestCase):
         av = self.assertValid
 
         # special case
-        if not is_jython:
+        if not support.is_jython:
             self.assertEqual(compile_command(""),
                              compile("pass", "<input>", 'single',
                                      PyCF_DONT_IMPLY_DEDENT))
@@ -122,7 +122,7 @@ class CodeopTests(unittest.TestCase):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
+    @support.skip_if_new_parser("Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
     def test_incomplete(self):
         ai = self.assertIncomplete
 
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 6535316dbe..a507ac0914 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -501,7 +501,7 @@ if 1:
         self.compile_single("if x:\n   f(x)\nelse:\n   g(x)")
         self.compile_single("class T:\n   pass")
 
-    @unittest.skipIf(sys.flags.use_peg, 'Pegen does not disallow multiline single stmts')
+    @support.skip_if_new_parser('Pegen does not disallow multiline single stmts')
     def test_bad_single_statement(self):
         self.assertInvalidSingle('1\n2')
         self.assertInvalidSingle('def f(): pass')
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 24ebc5ca9b..0bdfae1b7e 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -347,7 +347,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'isolated': 0,
         'use_environment': 1,
         'dev_mode': 0,
-        'use_peg': 1,
+        '_use_peg_parser': 1,
 
         'install_signal_handlers': 1,
         'use_hash_seed': 0,
@@ -729,7 +729,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'import_time': 1,
             'show_ref_count': 1,
             'malloc_stats': 1,
-            'use_peg': 0,
+            '_use_peg_parser': 0,
 
             'stdio_encoding': 'iso8859-1',
             'stdio_errors': 'replace',
@@ -792,6 +792,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_compat_env", config, preconfig,
                                api=API_COMPAT)
@@ -819,6 +820,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_python_env", config, preconfig,
                                api=API_PYTHON)
diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py
index bb1300c7c2..f8065788ce 100644
--- a/Lib/test/test_eof.py
+++ b/Lib/test/test_eof.py
@@ -26,7 +26,7 @@ class EOFTestCase(unittest.TestCase):
         else:
             raise support.TestFailed
 
-    @unittest.skipIf(sys.flags.use_peg, "TODO for PEG -- fails with new parser")
+    @support.skip_if_new_parser("TODO for PEG -- fails with new parser")
     def test_line_continuation_EOF(self):
         """A continuation at the end of input must be an error; bpo2180."""
         expect = 'unexpected EOF while parsing (<string>, line 1)'
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index c234c2b739..a207fb4863 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -178,7 +178,7 @@ class ExceptionTests(unittest.TestCase):
         s = '''if True:\n        print()\n\texec "mixed tabs and spaces"'''
         ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError)
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen column offsets might be different")
+    @support.skip_if_new_parser("Pegen column offsets might be different")
     def testSyntaxErrorOffset(self):
         def check(src, lineno, offset, encoding='utf-8'):
             with self.assertRaises(SyntaxError) as cm:
diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py
index 297a8aa90c..35ab934ab3 100644
--- a/Lib/test/test_flufl.py
+++ b/Lib/test/test_flufl.py
@@ -1,9 +1,10 @@
 import __future__
 import unittest
 import sys
+from test import support
 
 
-@unittest.skipIf(sys.flags.use_peg, "Not supported by pegen yet")
+@support.skip_if_new_parser("Not supported by pegen yet")
 class FLUFLTests(unittest.TestCase):
 
     def test_barry_as_bdfl(self):
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 802b08341e..8cafbe863c 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -12,6 +12,7 @@ import types
 import decimal
 import sys
 import unittest
+from test import support
 
 a_global = 'global variable'
 
@@ -206,7 +207,7 @@ f'{a * f"-{x()}-"}'"""
         call = binop.right.values[1].value
         self.assertEqual(type(call), ast.Call)
         self.assertEqual(call.lineno, 3)
-        if not sys.flags.use_peg:
+        if support.use_old_parser():
             self.assertEqual(call.col_offset, 11)
 
     def test_ast_line_numbers_duplicate_expression(self):
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
index 124a2790bf..0ee994f3b7 100644
--- a/Lib/test/test_parser.py
+++ b/Lib/test/test_parser.py
@@ -900,7 +900,7 @@ class ParserStackLimitTestCase(unittest.TestCase):
         st = parser.expr(e)
         st.compile()
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not trigger memory error with this many parenthesis")
+    @support.skip_if_new_parser("Pegen does not trigger memory error with this many parenthesis")
     def test_trigger_memory_error(self):
         e = self._nested_expression(100)
         rc, out, err = assert_python_failure('-Xoldparser', '-c', e)
diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py
index 5aa6c0d8f4..ea4afa6e17 100644
--- a/Lib/test/test_peg_parser.py
+++ b/Lib/test/test_peg_parser.py
@@ -6,6 +6,7 @@ import unittest
 from pathlib import PurePath
 from typing import Any, Union, Iterable, Tuple
 from textwrap import dedent
+from test import support
 
 
 TEST_CASES = [
@@ -720,7 +721,7 @@ class ASTGenerationTest(unittest.TestCase):
                 f"Actual error message does not match expexted for {source}"
             )
 
-    @unittest.skipIf(sys.flags.use_peg, "This tests nothing for now, since compile uses pegen as well")
+    @support.skip_if_new_parser("This tests nothing for now, since compile uses pegen as well")
     @unittest.expectedFailure
     def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
         for source in GOOD_BUT_FAIL_SOURCES:
diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py
index 332690051e..4d5fb6bc61 100644
--- a/Lib/test/test_positional_only_arg.py
+++ b/Lib/test/test_positional_only_arg.py
@@ -5,7 +5,7 @@ import pickle
 import unittest
 import sys
 
-from test.support import check_syntax_error
+from test.support import check_syntax_error, use_old_parser
 
 
 def global_pos_only_f(a, b, /):
@@ -24,7 +24,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
             compile(codestr + "\n", "<test>", "single")
 
     def test_invalid_syntax_errors(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument")
@@ -47,7 +47,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
         check_syntax_error(self, "def f(a, *, c, /, d, e): pass")
 
     def test_invalid_syntax_errors_async(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument")
@@ -236,7 +236,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
         self.assertEqual(x(1, 2), 3)
 
     def test_invalid_syntax_lambda(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument")
             check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument")
             check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument")
diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py
index 382c532df5..5a2fb8b372 100644
--- a/Lib/test/test_string_literals.py
+++ b/Lib/test/test_string_literals.py
@@ -33,6 +33,7 @@ import shutil
 import tempfile
 import unittest
 import warnings
+from test.support import check_syntax_warning, use_old_parser
 
 
 TEMPLATE = r"""# coding: %s
@@ -63,8 +64,6 @@ def byte(i):
 
 class TestLiterals(unittest.TestCase):
 
-    from test.support import check_syntax_warning
-
     def setUp(self):
         self.save_path = sys.path[:]
         self.tmpdir = tempfile.mkdtemp()
@@ -119,7 +118,7 @@ class TestLiterals(unittest.TestCase):
             eval("'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
@@ -129,7 +128,7 @@ class TestLiterals(unittest.TestCase):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(exc.lineno, 1)
 
     def test_eval_str_raw(self):
@@ -170,7 +169,7 @@ class TestLiterals(unittest.TestCase):
             eval("b'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
@@ -180,7 +179,7 @@ class TestLiterals(unittest.TestCase):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(exc.lineno, 1)
 
     def test_eval_bytes_raw(self):
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
index 4798f22b2b..aff8dd72b7 100644
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -678,8 +678,8 @@ class SyntaxTestCase(unittest.TestCase):
     def test_assign_call(self):
         self._check_error("f() = 1", "assign")
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not produce a specialized error "
-                                        "message yet")
+    @support.skip_if_new_parser("Pegen does not produce a specialized error "
+                                "message yet")
     def test_assign_del(self):
         self._check_error("del f()", "delete")
 
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index bd4ea47944..91a645b460 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -545,7 +545,7 @@ class SysModuleTest(unittest.TestCase):
     def test_sys_flags(self):
         self.assertTrue(sys.flags)
         attrs = ("debug",
-                 "inspect", "interactive", "optimize", "use_peg",
+                 "inspect", "interactive", "optimize",
                  "dont_write_bytecode", "no_user_site", "no_site",
                  "ignore_environment", "verbose", "bytes_warning", "quiet",
                  "hash_randomization", "isolated", "dev_mode", "utf8_mode")
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index 45f55e1f8a..7361d091cf 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -656,8 +656,7 @@ class BaseExceptionReportingTests:
         self.assertIn('inner_raise() # Marker', blocks[2])
         self.check_zero_div(blocks[2])
 
-    @unittest.skipIf(sys.flags.use_peg,
-                     "Pegen is arguably better here, so no need to fix this")
+    @support.skip_if_new_parser("Pegen is arguably better here, so no need to fix this")
     def test_syntax_error_offset_at_eol(self):
         # See #10186.
         def e():
diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py
index 80506e4b12..ce3b7985e6 100644
--- a/Lib/test/test_type_comments.py
+++ b/Lib/test/test_type_comments.py
@@ -1,6 +1,7 @@
 import ast
 import sys
 import unittest
+from test import support
 
 
 funcdef = """\
@@ -218,7 +219,7 @@ def favk(
 """
 
 
-@unittest.skipIf(sys.flags.use_peg, "Pegen does not support type comments yet")
+@support.skip_if_new_parser("Pegen does not support type comments yet")
 class TypeCommentTests(unittest.TestCase):
 
     lowest = 4  # Lowest minor version supported
diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py
index f5441ed54e..3bacd672d4 100644
--- a/Lib/test/test_unparse.py
+++ b/Lib/test/test_unparse.py
@@ -328,7 +328,7 @@ class UnparseTestCase(ASTTestCase):
             ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)"
         )
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not support type annotation yet")
+    @test.support.skip_if_new_parser("Pegen does not support type annotation yet")
     def test_function_type(self):
         for function_type in (
             "() -> int",