PEP 3147

14 files changed, 577 insertions, 170 deletions

diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py
index 144cf660e8..39874d97ec 100644
--- a/Lib/test/script_helper.py
+++ b/Lib/test/script_helper.py
@@ -11,6 +11,9 @@ import contextlib
 import shutil
 import zipfile
 
+from imp import source_from_cache
+from test.support import make_legacy_pyc
+
 # Executing the interpreter in a subprocess
 def python_exit_code(*args):
     cmd_line = [sys.executable, '-E']
@@ -62,20 +65,18 @@ def make_script(script_dir, script_basename, source):
     script_file.close()
     return script_name
 
-def compile_script(script_name):
-    py_compile.compile(script_name, doraise=True)
-    if __debug__:
-        compiled_name = script_name + 'c'
-    else:
-        compiled_name = script_name + 'o'
-    return compiled_name
-
 def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None):
     zip_filename = zip_basename+os.extsep+'zip'
     zip_name = os.path.join(zip_dir, zip_filename)
     zip_file = zipfile.ZipFile(zip_name, 'w')
     if name_in_zip is None:
-        name_in_zip = os.path.basename(script_name)
+        parts = script_name.split(os.sep)
+        if len(parts) >= 2 and parts[-2] == '__pycache__':
+            legacy_pyc = make_legacy_pyc(source_from_cache(script_name))
+            name_in_zip = os.path.basename(legacy_pyc)
+            script_name = legacy_pyc
+        else:
+            name_in_zip = os.path.basename(script_name)
     zip_file.write(script_name, name_in_zip)
     zip_file.close()
     #if test.test_support.verbose:
@@ -98,8 +99,8 @@ def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
     script_name = make_script(zip_dir, script_basename, source)
     unlink.append(script_name)
     if compiled:
-        init_name = compile_script(init_name)
-        script_name = compile_script(script_name)
+        init_name = py_compile(init_name, doraise=True)
+        script_name = py_compile(script_name, doraise=True)
         unlink.extend((init_name, script_name))
     pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)]
     script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name))
diff --git a/Lib/test/support.py b/Lib/test/support.py
index 9f9292d975..3c0002b0d0 100644
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -17,22 +17,25 @@ import unittest
 import importlib
 import collections
 import re
+import imp
 import time
 
-__all__ = ["Error", "TestFailed", "ResourceDenied", "import_module",
-           "verbose", "use_resources", "max_memuse", "record_original_stdout",
-           "get_original_stdout", "unload", "unlink", "rmtree", "forget",
-           "is_resource_enabled", "requires", "find_unused_port", "bind_port",
-           "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd",
-           "findfile", "sortdict", "check_syntax_error", "open_urlresource",
-           "check_warnings", "CleanImport", "EnvironmentVarGuard",
-           "TransientResource", "captured_output", "captured_stdout",
-           "time_out", "socket_peer_reset", "ioerror_peer_reset",
-           "run_with_locale",
-           "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner",
-           "run_unittest", "run_doctest", "threading_setup", "threading_cleanup",
-           "reap_children", "cpython_only", "check_impl_detail", "get_attribute",
-           "swap_item", "swap_attr"]
+__all__ = [
+    "Error", "TestFailed", "ResourceDenied", "import_module",
+    "verbose", "use_resources", "max_memuse", "record_original_stdout",
+    "get_original_stdout", "unload", "unlink", "rmtree", "forget",
+    "is_resource_enabled", "requires", "find_unused_port", "bind_port",
+    "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd",
+    "findfile", "sortdict", "check_syntax_error", "open_urlresource",
+    "check_warnings", "CleanImport", "EnvironmentVarGuard",
+    "TransientResource", "captured_output", "captured_stdout",
+    "time_out", "socket_peer_reset", "ioerror_peer_reset",
+    "run_with_locale", 'temp_umask',
+    "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner",
+    "run_unittest", "run_doctest", "threading_setup", "threading_cleanup",
+    "reap_children", "cpython_only", "check_impl_detail", "get_attribute",
+    "swap_item", "swap_attr",
+    ]
 
 
 class Error(Exception):
@@ -177,27 +180,50 @@ def unload(name):
 def unlink(filename):
     try:
         os.unlink(filename)
-    except OSError:
-        pass
+    except OSError as error:
+        # The filename need not exist.
+        if error.errno != errno.ENOENT:
+            raise
 
 def rmtree(path):
     try:
         shutil.rmtree(path)
-    except OSError as e:
+    except OSError as error:
         # Unix returns ENOENT, Windows returns ESRCH.
-        if e.errno not in (errno.ENOENT, errno.ESRCH):
+        if error.errno not in (errno.ENOENT, errno.ESRCH):
             raise
 
+def make_legacy_pyc(source):
+    """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
+
+    The choice of .pyc or .pyo extension is done based on the __debug__ flag
+    value.
+
+    :param source: The file system path to the source file.  The source file
+        does not need to exist, however the PEP 3147 pyc file must exist.
+    :return: The file system path to the legacy pyc file.
+    """
+    pyc_file = imp.cache_from_source(source)
+    up_one = os.path.dirname(os.path.abspath(source))
+    legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o'))
+    os.rename(pyc_file, legacy_pyc)
+    return legacy_pyc
+
 def forget(modname):
-    '''"Forget" a module was ever imported by removing it from sys.modules and
-    deleting any .pyc and .pyo files.'''
+    """'Forget' a module was ever imported.
+
+    This removes the module from sys.modules and deletes any PEP 3147 or
+    legacy .pyc and .pyo files.
+    """
     unload(modname)
     for dirname in sys.path:
-        unlink(os.path.join(dirname, modname + '.pyc'))
-        # Deleting the .pyo file cannot be within the 'try' for the .pyc since
-        # the chance exists that there is no .pyc (and thus the 'try' statement
-        # is exited) but there is a .pyo file.
-        unlink(os.path.join(dirname, modname + '.pyo'))
+        source = os.path.join(dirname, modname + '.py')
+        # It doesn't matter if they exist or not, unlink all possible
+        # combinations of PEP 3147 and legacy pyc and pyo files.
+        unlink(source + 'c')
+        unlink(source + 'o')
+        unlink(imp.cache_from_source(source, debug_override=True))
+        unlink(imp.cache_from_source(source, debug_override=False))
 
 def is_resource_enabled(resource):
     """Test whether a resource is enabled.  Known resources are set by
@@ -208,7 +234,9 @@ def requires(resource, msg=None):
     """Raise ResourceDenied if the specified resource is not available.
 
     If the caller's module is __main__ then automatically return True.  The
-    possibility of False being returned occurs when regrtest.py is executing."""
+    possibility of False being returned occurs when regrtest.py is
+    executing.
+    """
     # see if the caller's module is __main__ - if so, treat as if
     # the resource was set
     if sys._getframe(1).f_globals.get("__name__") == "__main__":
@@ -405,6 +433,16 @@ def temp_cwd(name='tempcwd', quiet=False):
             rmtree(name)
 
 
+@contextlib.contextmanager
+def temp_umask(umask):
+    """Context manager that temporarily sets the process umask."""
+    oldmask = os.umask(umask)
+    try:
+        yield
+    finally:
+        os.umask(oldmask)
+
+
 def findfile(file, here=__file__, subdir=None):
     """Try to find a file on sys.path and the working directory.  If it is not
     found the argument passed to the function is returned (this does not
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index f7c27a7315..3f4dd6d919 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -1,12 +1,14 @@
-# Tests command line execution of scripts
+# tests command line execution of scripts
 
 import unittest
 import os
 import os.path
+import py_compile
+
 import test.support
-from test.script_helper import (run_python,
-                                temp_dir, make_script, compile_script,
-                                make_pkg, make_zip_script, make_zip_pkg)
+from test.script_helper import (
+    make_pkg, make_script, make_zip_pkg, make_zip_script, run_python,
+    temp_dir)
 
 verbose = test.support.verbose
 
@@ -28,6 +30,7 @@ assertEqual(result, ['Top level assignment', 'Lower level reference'])
 # Check population of magic variables
 assertEqual(__name__, '__main__')
 print('__file__==%r' % __file__)
+assertEqual(__cached__, None)
 print('__package__==%r' % __package__)
 # Check the sys module
 import sys
@@ -101,9 +104,10 @@ class CmdLineTest(unittest.TestCase):
     def test_script_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, 'script')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
-            self._check_script(compiled_name, compiled_name, compiled_name, None)
+            self._check_script(compiled_name, compiled_name,
+                               compiled_name, None)
 
     def test_directory(self):
         with temp_dir() as script_dir:
@@ -113,9 +117,10 @@ class CmdLineTest(unittest.TestCase):
     def test_directory_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
-            self._check_script(script_dir, compiled_name, script_dir, '')
+            pyc_file = test.support.make_legacy_pyc(script_name)
+            self._check_script(script_dir, pyc_file, script_dir, '')
 
     def test_directory_error(self):
         with temp_dir() as script_dir:
@@ -131,7 +136,7 @@ class CmdLineTest(unittest.TestCase):
     def test_zipfile_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
             self._check_script(zip_name, run_name, zip_name, '')
 
@@ -176,11 +181,12 @@ class CmdLineTest(unittest.TestCase):
             pkg_dir = os.path.join(script_dir, 'test_pkg')
             make_pkg(pkg_dir)
             script_name = _make_test_script(pkg_dir, '__main__')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
+            pyc_file = test.support.make_legacy_pyc(script_name)
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
-            self._check_script(launch_name, compiled_name,
-                               compiled_name, 'test_pkg')
+            self._check_script(launch_name, pyc_file,
+                               pyc_file, 'test_pkg')
 
     def test_package_error(self):
         with temp_dir() as script_dir:
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 4b6feba24e..8b34587390 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -5,22 +5,23 @@ import os
 import py_compile
 import shutil
 import struct
+import subprocess
 import tempfile
-from test import support
 import unittest
 import io
 
+from test import support
 
 class CompileallTests(unittest.TestCase):
 
     def setUp(self):
         self.directory = tempfile.mkdtemp()
         self.source_path = os.path.join(self.directory, '_test.py')
-        self.bc_path = self.source_path + ('c' if __debug__ else 'o')
+        self.bc_path = imp.cache_from_source(self.source_path)
         with open(self.source_path, 'w') as file:
             file.write('x = 123\n')
         self.source_path2 = os.path.join(self.directory, '_test2.py')
-        self.bc_path2 = self.source_path2 + ('c' if __debug__ else 'o')
+        self.bc_path2 = imp.cache_from_source(self.source_path2)
         shutil.copyfile(self.source_path, self.source_path2)
 
     def tearDown(self):
@@ -65,17 +66,19 @@ class CompileallTests(unittest.TestCase):
             except:
                 pass
         compileall.compile_file(self.source_path, force=False, quiet=True)
-        self.assertTrue(os.path.isfile(self.bc_path) \
-                        and not os.path.isfile(self.bc_path2))
+        self.assertTrue(os.path.isfile(self.bc_path) and
+                        not os.path.isfile(self.bc_path2))
         os.unlink(self.bc_path)
         compileall.compile_dir(self.directory, force=False, quiet=True)
-        self.assertTrue(os.path.isfile(self.bc_path) \
-                        and os.path.isfile(self.bc_path2))
+        self.assertTrue(os.path.isfile(self.bc_path) and
+                        os.path.isfile(self.bc_path2))
         os.unlink(self.bc_path)
         os.unlink(self.bc_path2)
 
+
 class EncodingTest(unittest.TestCase):
-    'Issue 6716: compileall should escape source code when printing errors to stdout.'
+    """Issue 6716: compileall should escape source code when printing errors
+    to stdout."""
 
     def setUp(self):
         self.directory = tempfile.mkdtemp()
@@ -95,9 +98,65 @@ class EncodingTest(unittest.TestCase):
         finally:
             sys.stdout = orig_stdout
 
+class CommandLineTests(unittest.TestCase):
+    """Test some aspects of compileall's CLI."""
+
+    def setUp(self):
+        self.addCleanup(self._cleanup)
+        self.directory = tempfile.mkdtemp()
+        self.pkgdir = os.path.join(self.directory, 'foo')
+        os.mkdir(self.pkgdir)
+        # Touch the __init__.py and a package module.
+        with open(os.path.join(self.pkgdir, '__init__.py'), 'w'):
+            pass
+        with open(os.path.join(self.pkgdir, 'bar.py'), 'w'):
+            pass
+        sys.path.insert(0, self.directory)
+
+    def _cleanup(self):
+        support.rmtree(self.directory)
+        assert sys.path[0] == self.directory, 'Missing path'
+        del sys.path[0]
+
+    def test_pep3147_paths(self):
+        # Ensure that the default behavior of compileall's CLI is to create
+        # PEP 3147 pyc/pyo files.
+        retcode = subprocess.call(
+            (sys.executable, '-m', 'compileall', '-q', self.pkgdir))
+        self.assertEqual(retcode, 0)
+        # Verify the __pycache__ directory contents.
+        cachedir = os.path.join(self.pkgdir, '__pycache__')
+        self.assertTrue(os.path.exists(cachedir))
+        ext = ('pyc' if __debug__ else 'pyo')
+        expected = sorted(base.format(imp.get_tag(), ext) for base in
+                          ('__init__.{}.{}', 'bar.{}.{}'))
+        self.assertEqual(sorted(os.listdir(cachedir)), expected)
+        # Make sure there are no .pyc files in the source directory.
+        self.assertFalse([pyc_file for pyc_file in os.listdir(self.pkgdir)
+                          if pyc_file.endswith(ext)])
+
+    def test_legacy_paths(self):
+        # Ensure that with the proper switch, compileall leaves legacy
+        # pyc/pyo files, and no __pycache__ directory.
+        retcode = subprocess.call(
+            (sys.executable, '-m', 'compileall', '-b', '-q', self.pkgdir))
+        self.assertEqual(retcode, 0)
+        # Verify the __pycache__ directory contents.
+        cachedir = os.path.join(self.pkgdir, '__pycache__')
+        self.assertFalse(os.path.exists(cachedir))
+        ext = ('pyc' if __debug__ else 'pyo')
+        expected = [base.format(ext) for base in ('__init__.{}', 'bar.{}')]
+        expected.extend(['__init__.py', 'bar.py'])
+        expected.sort()
+        self.assertEqual(sorted(os.listdir(self.pkgdir)), expected)
+
+
 def test_main():
-    support.run_unittest(CompileallTests,
-                         EncodingTest)
+    support.run_unittest(
+        CommandLineTests,
+        CompileallTests,
+        EncodingTest,
+        )
 
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py
index 79cc1c340d..28186bbd4d 100644
--- a/Lib/test/test_frozen.py
+++ b/Lib/test/test_frozen.py
@@ -11,7 +11,7 @@ class FrozenTests(unittest.TestCase):
         except ImportError as x:
             self.fail("import __hello__ failed:" + str(x))
         self.assertEqual(__hello__.initialized, True)
-        self.assertEqual(len(dir(__hello__)), 6, dir(__hello__))
+        self.assertEqual(len(dir(__hello__)), 7, dir(__hello__))
 
         try:
             import __phello__
@@ -19,9 +19,9 @@ class FrozenTests(unittest.TestCase):
             self.fail("import __phello__ failed:" + str(x))
         self.assertEqual(__phello__.initialized, True)
         if not "__phello__.spam" in sys.modules:
-            self.assertEqual(len(dir(__phello__)), 7, dir(__phello__))
-        else:
             self.assertEqual(len(dir(__phello__)), 8, dir(__phello__))
+        else:
+            self.assertEqual(len(dir(__phello__)), 9, dir(__phello__))
         self.assertEquals(__phello__.__path__, [__phello__.__name__])
 
         try:
@@ -29,8 +29,8 @@ class FrozenTests(unittest.TestCase):
         except ImportError as x:
             self.fail("import __phello__.spam failed:" + str(x))
         self.assertEqual(__phello__.spam.initialized, True)
-        self.assertEqual(len(dir(__phello__.spam)), 6)
-        self.assertEqual(len(dir(__phello__)), 8)
+        self.assertEqual(len(dir(__phello__.spam)), 7)
+        self.assertEqual(len(dir(__phello__)), 9)
 
         try:
             import __phello__.foo
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index e995bf0630..6412f3f974 100644
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -1,6 +1,7 @@
 import imp
 import os
 import os.path
+import shutil
 import sys
 import unittest
 from test import support
@@ -139,7 +140,8 @@ class ImportTests(unittest.TestCase):
             mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
             self.assertEqual(mod.a, 1)
 
-            mod = imp.load_compiled(temp_mod_name, temp_mod_name + '.pyc')
+            mod = imp.load_compiled(
+                temp_mod_name, imp.cache_from_source(temp_mod_name + '.py'))
             self.assertEqual(mod.a, 1)
 
             if not os.path.exists(test_package_name):
@@ -184,11 +186,132 @@ class ReloadTests(unittest.TestCase):
             imp.reload(marshal)
 
 
+class PEP3147Tests(unittest.TestCase):
+    """Tests of PEP 3147."""
+
+    tag = imp.get_tag()
+
+    def test_cache_from_source(self):
+        # Given the path to a .py file, return the path to its PEP 3147
+        # defined .pyc file (i.e. under __pycache__).
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz/qux.py', True),
+            '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag))
+
+    def test_cache_from_source_optimized(self):
+        # Given the path to a .py file, return the path to its PEP 3147
+        # defined .pyo file (i.e. under __pycache__).
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz/qux.py', False),
+            '/foo/bar/baz/__pycache__/qux.{}.pyo'.format(self.tag))
+
+    def test_cache_from_source_cwd(self):
+        self.assertEqual(imp.cache_from_source('foo.py', True),
+                         os.sep.join(('__pycache__',
+                                      'foo.{}.pyc'.format(self.tag))))
+
+    def test_cache_from_source_override(self):
+        # When debug_override is not None, it can be any true-ish or false-ish
+        # value.
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz.py', []),
+            '/foo/bar/__pycache__/baz.{}.pyo'.format(self.tag))
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz.py', [17]),
+            '/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag))
+        # However if the bool-ishness can't be determined, the exception
+        # propagates.
+        class Bearish:
+            def __bool__(self): raise RuntimeError
+        self.assertRaises(
+            RuntimeError,
+            imp.cache_from_source, '/foo/bar/baz.py', Bearish())
+
+    @unittest.skipIf(os.altsep is None,
+                     'test meaningful only where os.altsep is defined')
+    def test_altsep_cache_from_source(self):
+        # Windows path and PEP 3147.
+        self.assertEqual(
+            imp.cache_from_source('\\foo\\bar\\baz\\qux.py', True),
+            '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+    @unittest.skipIf(os.altsep is None,
+                     'test meaningful only where os.altsep is defined')
+    def test_altsep_and_sep_cache_from_source(self):
+        # Windows path and PEP 3147 where altsep is right of sep.
+        self.assertEqual(
+            imp.cache_from_source('\\foo\\bar/baz\\qux.py', True),
+            '\\foo\\bar/baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+    @unittest.skipIf(os.altsep is None,
+                     'test meaningful only where os.altsep is defined')
+    def test_sep_altsep_and_sep_cache_from_source(self):
+        # Windows path and PEP 3147 where sep is right of altsep.
+        self.assertEqual(
+            imp.cache_from_source('\\foo\\bar\\baz/qux.py', True),
+            '\\foo\\bar\\baz/__pycache__/qux.{}.pyc'.format(self.tag))
+
+    def test_source_from_cache(self):
+        # Given the path to a PEP 3147 defined .pyc file, return the path to
+        # its source.  This tests the good path.
+        self.assertEqual(imp.source_from_cache(
+            '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)),
+            '/foo/bar/baz/qux.py')
+
+    def test_source_from_cache_bad_path(self):
+        # When the path to a pyc file is not in PEP 3147 format, a ValueError
+        # is raised.
+        self.assertRaises(
+            ValueError, imp.source_from_cache, '/foo/bar/bazqux.pyc')
+
+    def test_source_from_cache_no_slash(self):
+        # No slashes at all in path -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache, 'foo.cpython-32.pyc')
+
+    def test_source_from_cache_too_few_dots(self):
+        # Too few dots in final path component -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache, '__pycache__/foo.pyc')
+
+    def test_source_from_cache_too_many_dots(self):
+        # Too many dots in final path component -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache,
+            '__pycache__/foo.cpython-32.foo.pyc')
+
+    def test_source_from_cache_no__pycache__(self):
+        # Another problem with the path -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache,
+            '/foo/bar/foo.cpython-32.foo.pyc')
+
+    def test_package___file__(self):
+        # Test that a package's __file__ points to the right source directory.
+        os.mkdir('pep3147')
+        sys.path.insert(0, os.curdir)
+        def cleanup():
+            if sys.path[0] == os.curdir:
+                del sys.path[0]
+            shutil.rmtree('pep3147')
+        self.addCleanup(cleanup)
+        # Touch the __init__.py file.
+        with open('pep3147/__init__.py', 'w'):
+            pass
+        m = __import__('pep3147')
+        # Ensure we load the pyc file.
+        support.forget('pep3147')
+        m = __import__('pep3147')
+        self.assertEqual(m.__file__,
+                         os.sep.join(('.', 'pep3147', '__init__.py')))
+
+
 def test_main():
     tests = [
         ImportTests,
+        PEP3147Tests,
         ReloadTests,
-    ]
+        ]
     try:
         import _thread
     except ImportError:
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
index 9b34467ab0..0a21e182eb 100644
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -1,4 +1,5 @@
 import builtins
+import errno
 import imp
 import marshal
 import os
@@ -8,8 +9,11 @@ import shutil
 import stat
 import sys
 import unittest
-from test.support import (unlink, TESTFN, unload, run_unittest, is_jython,
-                          check_warnings, EnvironmentVarGuard, swap_attr, swap_item)
+
+from test.support import (
+    EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython,
+    make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask,
+    unlink, unload)
 
 
 def remove_files(name):
@@ -19,12 +23,18 @@ def remove_files(name):
               name + ".pyw",
               name + "$py.class"):
         unlink(f)
+    try:
+        shutil.rmtree('__pycache__')
+    except OSError as error:
+        if error.errno != errno.ENOENT:
+            raise
 
 
 class ImportTests(unittest.TestCase):
 
     def tearDown(self):
         unload(TESTFN)
+
     setUp = tearDown
 
     def test_case_sensitivity(self):
@@ -53,8 +63,8 @@ class ImportTests(unittest.TestCase):
                 pyc = TESTFN + ".pyc"
 
             with open(source, "w") as f:
-                print("# This tests Python's ability to import a", ext, "file.",
-                      file=f)
+                print("# This tests Python's ability to import a",
+                      ext, "file.", file=f)
                 a = random.randrange(1000)
                 b = random.randrange(1000)
                 print("a =", a, file=f)
@@ -73,10 +83,10 @@ class ImportTests(unittest.TestCase):
                 self.assertEqual(mod.b, b,
                     "module loaded (%s) but contents invalid" % mod)
             finally:
+                forget(TESTFN)
                 unlink(source)
                 unlink(pyc)
                 unlink(pyo)
-                unload(TESTFN)
 
         sys.path.insert(0, os.curdir)
         try:
@@ -87,32 +97,31 @@ class ImportTests(unittest.TestCase):
         finally:
             del sys.path[0]
 
-    @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems")
+    @unittest.skipUnless(os.name == 'posix',
+                         "test meaningful only on posix systems")
     def test_execute_bit_not_copied(self):
         # Issue 6070: under posix .pyc files got their execute bit set if
         # the .py file had the execute bit set, but they aren't executable.
-        oldmask = os.umask(0o022)
-        sys.path.insert(0, os.curdir)
-        try:
-            fname = TESTFN + os.extsep + "py"
-            f = open(fname, 'w').close()
-            os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
-                             stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
-            __import__(TESTFN)
-            fn = fname + 'c'
-            if not os.path.exists(fn):
-                fn = fname + 'o'
+        with temp_umask(0o022):
+            sys.path.insert(0, os.curdir)
+            try:
+                fname = TESTFN + os.extsep + "py"
+                f = open(fname, 'w').close()
+                os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
+                                 stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
+                __import__(TESTFN)
+                fn = imp.cache_from_source(fname)
                 if not os.path.exists(fn):
                     self.fail("__import__ did not result in creation of "
                               "either a .pyc or .pyo file")
-            s = os.stat(fn)
-            self.assertEqual(stat.S_IMODE(s.st_mode),
-                             stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
-        finally:
-            os.umask(oldmask)
-            remove_files(TESTFN)
-            unload(TESTFN)
-            del sys.path[0]
+                    s = os.stat(fn)
+                    self.assertEqual(
+                        stat.S_IMODE(s.st_mode),
+                        stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+            finally:
+                del sys.path[0]
+                remove_files(TESTFN)
+                unload(TESTFN)
 
     def test_imp_module(self):
         # Verify that the imp module can correctly load and find .py files
@@ -144,10 +153,12 @@ class ImportTests(unittest.TestCase):
                 f.write('"",\n')
             f.write(']')
 
-        # Compile & remove .py file, we only need .pyc (or .pyo).
+        # Compile & remove .py file, we only need .pyc (or .pyo), but that
+        # must be relocated to the PEP 3147 bytecode-only location.
         with open(filename, 'r') as f:
             py_compile.compile(filename)
         unlink(filename)
+        make_legacy_pyc(filename)
 
         # Need to be able to load from current dir.
         sys.path.append('')
@@ -247,8 +258,9 @@ class ImportTests(unittest.TestCase):
             self.assertTrue(mod.__file__.endswith('.py'))
             os.remove(source)
             del sys.modules[TESTFN]
+            make_legacy_pyc(source)
             mod = __import__(TESTFN)
-            ext = mod.__file__[-4:]
+            base, ext = os.path.splitext(mod.__file__)
             self.assertIn(ext, ('.pyc', '.pyo'))
         finally:
             del sys.path[0]
@@ -298,7 +310,7 @@ func_filename = func.__code__.co_filename
 """
     dir_name = os.path.abspath(TESTFN)
     file_name = os.path.join(dir_name, module_name) + os.extsep + "py"
-    compiled_name = file_name + ("c" if __debug__ else "o")
+    compiled_name = imp.cache_from_source(file_name)
 
     def setUp(self):
         self.sys_path = sys.path[:]
@@ -346,8 +358,9 @@ func_filename = func.__code__.co_filename
         target = "another_module.py"
         py_compile.compile(self.file_name, dfile=target)
         os.remove(self.file_name)
+        pyc_file = make_legacy_pyc(self.file_name)
         mod = self.import_module()
-        self.assertEqual(mod.module_filename, self.compiled_name)
+        self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
         self.assertEqual(mod.func_filename, target)
 
@@ -476,10 +489,143 @@ class OverridingImportBuiltinTests(unittest.TestCase):
             self.assertEqual(foo(), os)
 
 
+class PycacheTests(unittest.TestCase):
+    # Test the various PEP 3147 related behaviors.
+
+    tag = imp.get_tag()
+
+    def _clean(self):
+        forget(TESTFN)
+        rmtree('__pycache__')
+        unlink(self.source)
+
+    def setUp(self):
+        self.source = TESTFN + '.py'
+        self._clean()
+        with open(self.source, 'w') as fp:
+            print('# This is a test file written by test_import.py', file=fp)
+        sys.path.insert(0, os.curdir)
+
+    def tearDown(self):
+        assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]'
+        del sys.path[0]
+        self._clean()
+
+    def test_import_pyc_path(self):
+        self.assertFalse(os.path.exists('__pycache__'))
+        __import__(TESTFN)
+        self.assertTrue(os.path.exists('__pycache__'))
+        self.assertTrue(os.path.exists(os.path.join(
+            '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag))))
+
+    @unittest.skipUnless(os.name == 'posix',
+                         "test meaningful only on posix systems")
+    def test_unwritable_directory(self):
+        # When the umask causes the new __pycache__ directory to be
+        # unwritable, the import still succeeds but no .pyc file is written.
+        with temp_umask(0o222):
+            __import__(TESTFN)
+        self.assertTrue(os.path.exists('__pycache__'))
+        self.assertFalse(os.path.exists(os.path.join(
+            '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag))))
+
+    def test_missing_source(self):
+        # With PEP 3147 cache layout, removing the source but leaving the pyc
+        # file does not satisfy the import.
+        __import__(TESTFN)
+        pyc_file = imp.cache_from_source(self.source)
+        self.assertTrue(os.path.exists(pyc_file))
+        os.remove(self.source)
+        forget(TESTFN)
+        self.assertRaises(ImportError, __import__, TESTFN)
+
+    def test_missing_source_legacy(self):
+        # Like test_missing_source() except that for backward compatibility,
+        # when the pyc file lives where the py file would have been (and named
+        # without the tag), it is importable.  The __file__ of the imported
+        # module is the pyc location.
+        __import__(TESTFN)
+        # pyc_file gets removed in _clean() via tearDown().
+        pyc_file = make_legacy_pyc(self.source)
+        os.remove(self.source)
+        unload(TESTFN)
+        m = __import__(TESTFN)
+        self.assertEqual(m.__file__,
+                         os.path.join(os.curdir, os.path.relpath(pyc_file)))
+
+    def test___cached__(self):
+        # Modules now also have an __cached__ that points to the pyc file.
+        m = __import__(TESTFN)
+        pyc_file = imp.cache_from_source(TESTFN + '.py')
+        self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file))
+
+    def test___cached___legacy_pyc(self):
+        # Like test___cached__() except that for backward compatibility,
+        # when the pyc file lives where the py file would have been (and named
+        # without the tag), it is importable.  The __cached__ of the imported
+        # module is the pyc location.
+        __import__(TESTFN)
+        # pyc_file gets removed in _clean() via tearDown().
+        pyc_file = make_legacy_pyc(self.source)
+        os.remove(self.source)
+        unload(TESTFN)
+        m = __import__(TESTFN)
+        self.assertEqual(m.__cached__,
+                         os.path.join(os.curdir, os.path.relpath(pyc_file)))
+
+    def test_package___cached__(self):
+        # Like test___cached__ but for packages.
+        def cleanup():
+            shutil.rmtree('pep3147')
+        os.mkdir('pep3147')
+        self.addCleanup(cleanup)
+        # Touch the __init__.py
+        with open(os.path.join('pep3147', '__init__.py'), 'w'):
+            pass
+        with open(os.path.join('pep3147', 'foo.py'), 'w'):
+            pass
+        unload('pep3147.foo')
+        unload('pep3147')
+        m = __import__('pep3147.foo')
+        init_pyc = imp.cache_from_source(
+            os.path.join('pep3147', '__init__.py'))
+        self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
+        foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+        self.assertEqual(sys.modules['pep3147.foo'].__cached__,
+                         os.path.join(os.curdir, foo_pyc))
+
+    def test_package___cached___from_pyc(self):
+        # Like test___cached__ but ensuring __cached__ when imported from a
+        # PEP 3147 pyc file.
+        def cleanup():
+            shutil.rmtree('pep3147')
+        os.mkdir('pep3147')
+        self.addCleanup(cleanup)
+        unload('pep3147.foo')
+        unload('pep3147')
+        # Touch the __init__.py
+        with open(os.path.join('pep3147', '__init__.py'), 'w'):
+            pass
+        with open(os.path.join('pep3147', 'foo.py'), 'w'):
+            pass
+        m = __import__('pep3147.foo')
+        unload('pep3147.foo')
+        unload('pep3147')
+        m = __import__('pep3147.foo')
+        init_pyc = imp.cache_from_source(
+            os.path.join('pep3147', '__init__.py'))
+        self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
+        foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+        self.assertEqual(sys.modules['pep3147.foo'].__cached__,
+                         os.path.join(os.curdir, foo_pyc))
+
+
 def test_main(verbose=None):
-    run_unittest(ImportTests, PycRewritingTests, PathsTests, RelativeImportTests,
+    run_unittest(ImportTests, PycacheTests,
+                 PycRewritingTests, PathsTests, RelativeImportTests,
                  OverridingImportBuiltinTests)
 
+
 if __name__ == '__main__':
     # Test needs to be a package, so we can do relative imports.
     from test.test_import import test_main
diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py
index 2c195898fe..a342f7a7ae 100644
--- a/Lib/test/test_pkg.py
+++ b/Lib/test/test_pkg.py
@@ -196,14 +196,14 @@ class TestPkg(unittest.TestCase):
 
         import t5
         self.assertEqual(fixdir(dir(t5)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                           '__package__', '__path__', 'foo', 'string', 't5'])
         self.assertEqual(fixdir(dir(t5.foo)),
-                         ['__doc__', '__file__', '__name__', '__package__',
-                          'string'])
+                         ['__cached__', '__doc__', '__file__', '__name__',
+                          '__package__', 'string'])
         self.assertEqual(fixdir(dir(t5.string)),
-                         ['__doc__', '__file__', '__name__','__package__',
-                          'spam'])
+                         ['__cached__', '__doc__', '__file__', '__name__',
+                          '__package__', 'spam'])
 
     def test_6(self):
         hier = [
@@ -218,13 +218,13 @@ class TestPkg(unittest.TestCase):
 
         import t6
         self.assertEqual(fixdir(dir(t6)),
-                         ['__all__', '__doc__', '__file__',
+                         ['__all__', '__cached__', '__doc__', '__file__',
                           '__name__', '__package__', '__path__'])
         s = """
             import t6
             from t6 import *
             self.assertEqual(fixdir(dir(t6)),
-                             ['__all__', '__doc__', '__file__',
+                             ['__all__', '__cached__', '__doc__', '__file__',
                               '__name__', '__package__', '__path__',
                               'eggs', 'ham', 'spam'])
             self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
@@ -252,18 +252,18 @@ class TestPkg(unittest.TestCase):
         t7, sub, subsub = None, None, None
         import t7 as tas
         self.assertEqual(fixdir(dir(tas)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                           '__package__', '__path__'])
         self.assertFalse(t7)
         from t7 import sub as subpar
         self.assertEqual(fixdir(dir(subpar)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                           '__package__', '__path__'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         from t7.sub import subsub as subsubsub
         self.assertEqual(fixdir(dir(subsubsub)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                          '__package__', '__path__', 'spam'])
         self.assertFalse(t7)
         self.assertFalse(sub)
diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py
index a9a475c3c6..eab66fb158 100644
--- a/Lib/test/test_pkgimport.py
+++ b/Lib/test/test_pkgimport.py
@@ -1,5 +1,12 @@
-import os, sys, string, random, tempfile, unittest
-
+import os
+import sys
+import shutil
+import string
+import random
+import tempfile
+import unittest
+
+from imp import cache_from_source
 from test.support import run_unittest
 
 class TestImport(unittest.TestCase):
@@ -26,22 +33,17 @@ class TestImport(unittest.TestCase):
         self.module_path = os.path.join(self.package_dir, 'foo.py')
 
     def tearDown(self):
-        for file in os.listdir(self.package_dir):
-            os.remove(os.path.join(self.package_dir, file))
-        os.rmdir(self.package_dir)
-        os.rmdir(self.test_dir)
+        shutil.rmtree(self.test_dir)
         self.assertNotEqual(sys.path.count(self.test_dir), 0)
         sys.path.remove(self.test_dir)
         self.remove_modules()
 
     def rewrite_file(self, contents):
-        for extension in "co":
-            compiled_path = self.module_path + extension
-            if os.path.exists(compiled_path):
-                os.remove(compiled_path)
-        f = open(self.module_path, 'w')
-        f.write(contents)
-        f.close()
+        compiled_path = cache_from_source(self.module_path)
+        if os.path.exists(compiled_path):
+            os.remove(compiled_path)
+        with open(self.module_path, 'w') as f:
+            f.write(contents)
 
     def test_package_import__semantics(self):
 
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
index d0b81e35d8..603755ad0e 100644
--- a/Lib/test/test_pydoc.py
+++ b/Lib/test/test_pydoc.py
@@ -19,8 +19,7 @@ from test import pydoc_mod
 if hasattr(pydoc_mod, "__loader__"):
     del pydoc_mod.__loader__
 
-expected_text_pattern = \
-"""
+expected_text_pattern = """
 NAME
     test.pydoc_mod - This is a test module for test_pydoc
 
@@ -87,8 +86,7 @@ CREDITS
     Nobody
 """.strip()
 
-expected_html_pattern = \
-"""
+expected_html_pattern = """
 <table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="heading">
 <tr bgcolor="#7799ee">
 <td valign=bottom>&nbsp;<br>
@@ -186,7 +184,7 @@ war</tt></dd></dl>
 \x20\x20\x20\x20
 <tr><td bgcolor="#7799ee"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
 <td width="100%%">Nobody</td></tr></table>
-""".strip()
+""".strip() # ' <- emacs turd
 
 
 # output pattern for missing module
@@ -287,7 +285,8 @@ class PyDocDocTest(unittest.TestCase):
             ('i_am_not_here', 'i_am_not_here'),
             ('test.i_am_not_here_either', 'i_am_not_here_either'),
             ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
-            ('i_am_not_here.{}'.format(modname), 'i_am_not_here.{}'.format(modname)),
+            ('i_am_not_here.{}'.format(modname),
+             'i_am_not_here.{}'.format(modname)),
             ('test.{}'.format(modname), modname),
             )
 
@@ -304,9 +303,8 @@ class PyDocDocTest(unittest.TestCase):
             fullmodname = os.path.join(TESTFN, modname)
             sourcefn = fullmodname + os.extsep + "py"
             for importstring, expectedinmsg in testpairs:
-                f = open(sourcefn, 'w')
-                f.write("import {}\n".format(importstring))
-                f.close()
+                with open(sourcefn, 'w') as f:
+                    f.write("import {}\n".format(importstring))
                 try:
                     result = run_pydoc(modname).decode("ascii")
                 finally:
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index 995c8917d5..068eca94a0 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -5,9 +5,10 @@ import os.path
 import sys
 import re
 import tempfile
-from test.support import verbose, run_unittest, forget
-from test.script_helper import (temp_dir, make_script, compile_script,
-                                make_pkg, make_zip_script, make_zip_pkg)
+import py_compile
+from test.support import forget, make_legacy_pyc, run_unittest, verbose
+from test.script_helper import (
+    make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
 
 
 from runpy import _run_code, _run_module_code, run_module, run_path
@@ -45,6 +46,7 @@ class RunModuleCodeTest(unittest.TestCase):
         self.assertEqual(d["result"], self.expected_result)
         self.assertIs(d["__name__"], None)
         self.assertIs(d["__file__"], None)
+        self.assertIs(d["__cached__"], None)
         self.assertIs(d["__loader__"], None)
         self.assertIs(d["__package__"], None)
         self.assertIs(d["run_argv0"], saved_argv0)
@@ -73,6 +75,7 @@ class RunModuleCodeTest(unittest.TestCase):
         self.assertTrue(d2["run_name_in_sys_modules"])
         self.assertTrue(d2["module_in_sys_modules"])
         self.assertIs(d2["__file__"], file)
+        self.assertIs(d2["__cached__"], None)
         self.assertIs(d2["run_argv0"], file)
         self.assertIs(d2["__loader__"], loader)
         self.assertIs(d2["__package__"], package)
@@ -170,6 +173,7 @@ class RunModuleTest(unittest.TestCase):
             del d1 # Ensure __loader__ entry doesn't keep file open
             __import__(mod_name)
             os.remove(mod_fname)
+            make_legacy_pyc(mod_fname)
             if verbose: print("Running from compiled:", mod_name)
             d2 = run_module(mod_name) # Read from bytecode
             self.assertIn("x", d2)
@@ -192,6 +196,7 @@ class RunModuleTest(unittest.TestCase):
             del d1 # Ensure __loader__ entry doesn't keep file open
             __import__(mod_name)
             os.remove(mod_fname)
+            make_legacy_pyc(mod_fname)
             if verbose: print("Running from compiled:", pkg_name)
             d2 = run_module(pkg_name) # Read from bytecode
             self.assertIn("x", d2)
@@ -246,6 +251,7 @@ from ..uncle.cousin import nephew
             del d1 # Ensure __loader__ entry doesn't keep file open
             __import__(mod_name)
             os.remove(mod_fname)
+            make_legacy_pyc(mod_fname)
             if verbose: print("Running from compiled:", mod_name)
             d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
             self.assertIn("__package__", d2)
@@ -313,6 +319,7 @@ argv0 = sys.argv[0]
         result = run_path(script_name)
         self.assertEqual(result["__name__"], expected_name)
         self.assertEqual(result["__file__"], expected_file)
+        self.assertEqual(result["__cached__"], None)
         self.assertIn("argv0", result)
         self.assertEqual(result["argv0"], expected_argv0)
         self.assertEqual(result["__package__"], expected_package)
@@ -332,7 +339,7 @@ argv0 = sys.argv[0]
         with temp_dir() as script_dir:
             mod_name = 'script'
             script_name = self._make_test_script(script_dir, mod_name)
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
             self._check_script(compiled_name, "<run_path>", compiled_name,
                                compiled_name, None)
@@ -348,9 +355,10 @@ argv0 = sys.argv[0]
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
-            self._check_script(script_dir, "<run_path>", compiled_name,
+            legacy_pyc = make_legacy_pyc(script_name)
+            self._check_script(script_dir, "<run_path>", legacy_pyc,
                                script_dir, '')
 
     def test_directory_error(self):
@@ -371,8 +379,9 @@ argv0 = sys.argv[0]
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
-            compiled_name = compile_script(script_name)
-            zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
+            zip_name, fname = make_zip_script(script_dir, 'test_zip',
+                                              compiled_name)
             self._check_script(zip_name, "<run_path>", fname, zip_name, '')
 
     def test_zipfile_error(self):
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index 931a166f1c..1a50f19133 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -258,19 +258,38 @@ class ImportSideEffectTests(unittest.TestCase):
         """Restore sys.path"""
         sys.path[:] = self.sys_path
 
-    def test_abs__file__(self):
-        # Make sure all imported modules have their __file__ attribute
-        # as an absolute path.
-        # Handled by abs__file__()
-        site.abs__file__()
-        for module in (sys, os, builtins):
-            try:
-                self.assertTrue(os.path.isabs(module.__file__), repr(module))
-            except AttributeError:
-                continue
-        # We could try everything in sys.modules; however, when regrtest.py
-        # runs something like test_frozen before test_site, then we will
-        # be testing things loaded *after* test_site did path normalization
+    def test_abs_paths(self):
+        # Make sure all imported modules have their __file__ and __cached__
+        # attributes as absolute paths.  Arranging to put the Lib directory on
+        # PYTHONPATH would cause the os module to have a relative path for
+        # __file__ if abs_paths() does not get run.  sys and builtins (the
+        # only other modules imported before site.py runs) do not have
+        # __file__ or __cached__ because they are built-in.
+        parent = os.path.relpath(os.path.dirname(os.__file__))
+        env = os.environ.copy()
+        env['PYTHONPATH'] = parent
+        command = 'import os; print(os.__file__, os.__cached__)'
+        # First, prove that with -S (no 'import site'), the paths are
+        # relative.
+        proc = subprocess.Popen([sys.executable, '-S', '-c', command],
+                                env=env,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        stdout, stderr = proc.communicate()
+        self.assertEqual(proc.returncode, 0)
+        os__file__, os__cached__ = stdout.split()
+        self.assertFalse(os.path.isabs(os__file__))
+        self.assertFalse(os.path.isabs(os__cached__))
+        # Now, with 'import site', it works.
+        proc = subprocess.Popen([sys.executable, '-c', command],
+                                env=env,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        stdout, stderr = proc.communicate()
+        self.assertEqual(proc.returncode, 0)
+        os__file__, os__cached__ = stdout.split()
+        self.assertTrue(os.path.isabs(os__file__))
+        self.assertTrue(os.path.isabs(os__cached__))
 
     def test_no_duplicate_paths(self):
         # No duplicate paths should exist in sys.path
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 8e2cf553e7..e9a90e55fb 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -6,6 +6,7 @@ except ImportError:
 
 import io
 import os
+import imp
 import time
 import shutil
 import struct
@@ -587,7 +588,13 @@ class PyZipFileTests(unittest.TestCase):
         with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp:
             fn = __file__
             if fn.endswith('.pyc') or fn.endswith('.pyo'):
-                fn = fn[:-1]
+                path_split = fn.split(os.sep)
+                if os.altsep is not None:
+                    path_split.extend(fn.split(os.altsep))
+                if '__pycache__' in path_split:
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
 
             zipfp.writepy(fn)
 
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index c89aef5799..ba4e34a960 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -48,17 +48,14 @@ NOW = time.time()
 test_pyc = make_pyc(test_co, NOW)
 
 
-if __debug__:
-    pyc_ext = ".pyc"
-else:
-    pyc_ext = ".pyo"
-
-
 TESTMOD = "ziptestmodule"
 TESTPACK = "ziptestpackage"
 TESTPACK2 = "ziptestpackage2"
 TEMP_ZIP = os.path.abspath("junk95142.zip")
 
+pyc_file = imp.cache_from_source(TESTMOD + '.py')
+pyc_ext = ('.pyc' if __debug__ else '.pyo')
+
 
 class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
 
@@ -83,14 +80,11 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             stuff = kw.get("stuff", None)
             if stuff is not None:
                 # Prepend 'stuff' to the start of the zipfile
-                f = open(TEMP_ZIP, "rb")
-                data = f.read()
-                f.close()
-
-                f = open(TEMP_ZIP, "wb")
-                f.write(stuff)
-                f.write(data)
-                f.close()
+                with open(TEMP_ZIP, "rb") as f:
+                    data = f.read()
+                with open(TEMP_ZIP, "wb") as f:
+                    f.write(stuff)
+                    f.write(data)
 
             sys.path.insert(0, TEMP_ZIP)
 
@@ -180,8 +174,9 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
 
     def testBadMTime(self):
         badtime_pyc = bytearray(test_pyc)
-        badtime_pyc[7] ^= 0x02  # flip the second bit -- not the first as that one
-                                # isn't stored in the .py's mtime in the zip archive.
+        # flip the second bit -- not the first as that one isn't stored in the
+        # .py's mtime in the zip archive.
+        badtime_pyc[7] ^= 0x02
         files = {TESTMOD + ".py": (NOW, test_src),
                  TESTMOD + pyc_ext: (NOW, badtime_pyc)}
         self.doTest(".py", files, TESTMOD)
@@ -232,7 +227,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             self.assertEquals(zi.get_source(TESTPACK), None)
             self.assertEquals(zi.get_source(mod_path), None)
             self.assertEquals(zi.get_filename(mod_path), mod.__file__)
-            # To pass in the module name instead of the path, we must use the right importer
+            # To pass in the module name instead of the path, we must use the
+            # right importer
             loader = mod.__loader__
             self.assertEquals(loader.get_source(mod_name), None)
             self.assertEquals(loader.get_filename(mod_name), mod.__file__)
@@ -266,8 +262,10 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             mod = zi.load_module(TESTPACK2)
             self.assertEquals(zi.get_filename(TESTPACK2), mod.__file__)
 
-            self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
-            self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
+            self.assertEquals(
+                zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
+            self.assertEquals(
+                zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
 
             mod_path = TESTPACK2 + os.sep + TESTMOD
             mod_name = module_path_to_dotted_name(mod_path)
@@ -276,7 +274,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             self.assertEquals(zi.get_source(TESTPACK2), None)
             self.assertEquals(zi.get_source(mod_path), None)
             self.assertEquals(zi.get_filename(mod_path), mod.__file__)
-            # To pass in the module name instead of the path, we must use the right importer
+            # To pass in the module name instead of the path, we must use the
+            # right importer
             loader = mod.__loader__
             self.assertEquals(loader.get_source(mod_name), None)
             self.assertEquals(loader.get_filename(mod_name), mod.__file__)