Merged revisions 67900-67901,67919,67928,67984,67991-67993,68106-68108,68110 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r67900 | benjamin.peterson | 2008-12-22 14:02:45 -0600 (Mon, 22 Dec 2008) | 4 lines

  fix_execfile: wrap the open(fn).read() call in compile(), so the filename is preserved

  also add unittests for the fixer
........
  r67901 | benjamin.peterson | 2008-12-22 14:09:55 -0600 (Mon, 22 Dec 2008) | 1 line

  remove unused import
........
  r67919 | benjamin.peterson | 2008-12-23 13:12:22 -0600 (Tue, 23 Dec 2008) | 1 line

  copy permission bits from the backup to the original
........
  r67928 | benjamin.peterson | 2008-12-26 20:49:30 -0600 (Fri, 26 Dec 2008) | 1 line

  don't be so idiot about multiple local imports in fix_import; still won't handle absolute and local imports on the same line
........
  r67984 | benjamin.peterson | 2008-12-28 09:55:16 -0600 (Sun, 28 Dec 2008) | 1 line

  don't need loop
........
  r67991 | benjamin.peterson | 2008-12-28 14:30:26 -0600 (Sun, 28 Dec 2008) | 1 line

  actually call finish_tree()
........
  r67992 | benjamin.peterson | 2008-12-28 14:34:47 -0600 (Sun, 28 Dec 2008) | 1 line

  remove useless test
........
  r67993 | benjamin.peterson | 2008-12-28 15:04:32 -0600 (Sun, 28 Dec 2008) | 1 line

  update pyk3's test grammar
........
  r68106 | benjamin.peterson | 2008-12-31 11:53:58 -0600 (Wed, 31 Dec 2008) | 1 line

  #2734 don't convert every instance of long (eg if it's an attribute)
........
  r68107 | benjamin.peterson | 2008-12-31 11:55:10 -0600 (Wed, 31 Dec 2008) | 1 line

  add another test
........
  r68108 | benjamin.peterson | 2008-12-31 12:00:12 -0600 (Wed, 31 Dec 2008) | 1 line

  don't change long even if it's the only argument name
........
  r68110 | benjamin.peterson | 2008-12-31 14:13:26 -0600 (Wed, 31 Dec 2008) | 1 line

  remove unused import
........

4 files changed, 84 insertions, 51 deletions

diff --git a/Lib/lib2to3/fixes/fix_execfile.py b/Lib/lib2to3/fixes/fix_execfile.py
index 5854900d2f..f7a7a70014 100644
--- a/Lib/lib2to3/fixes/fix_execfile.py
+++ b/Lib/lib2to3/fixes/fix_execfile.py
@@ -7,9 +7,9 @@ This converts usages of the execfile function into calls to the built-in
 exec() function.
 """
 
-from .. import pytree
 from .. import fixer_base
-from ..fixer_util import Comma, Name, Call, LParen, RParen, Dot
+from ..fixer_util import (Comma, Name, Call, LParen, RParen, Dot, Node,
+                          ArgList, String, syms)
 
 
 class FixExecfile(fixer_base.BaseFix):
@@ -22,16 +22,30 @@ class FixExecfile(fixer_base.BaseFix):
 
     def transform(self, node, results):
         assert results
-        syms = self.syms
         filename = results["filename"]
         globals = results.get("globals")
         locals = results.get("locals")
-        args = [Name('open'), LParen(), filename.clone(), RParen(), Dot(),
-                Name('read'), LParen(), RParen()]
-        args[0].set_prefix("")
+
+        # Copy over the prefix from the right parentheses end of the execfile
+        # call.
+        execfile_paren = node.children[-1].children[-1].clone()
+        # Construct open().read().
+        open_args = ArgList([filename.clone()], rparen=execfile_paren)
+        open_call = Node(syms.power, [Name("open"), open_args])
+        read = [Node(syms.trailer, [Dot(), Name('read')]),
+                Node(syms.trailer, [LParen(), RParen()])]
+        open_expr = [open_call] + read
+        # Wrap the open call in a compile call. This is so the filename will be
+        # preserved in the execed code.
+        filename_arg = filename.clone()
+        filename_arg.set_prefix(" ")
+        exec_str = String("'exec'", " ")
+        compile_args = open_expr + [Comma(), filename_arg, Comma(), exec_str]
+        compile_call = Call(Name("compile"), compile_args, "")
+        # Finally, replace the execfile call with an exec call.
+        args = [compile_call]
         if globals is not None:
             args.extend([Comma(), globals.clone()])
         if locals is not None:
             args.extend([Comma(), locals.clone()])
-
         return Call(Name("exec"), args, prefix=node.get_prefix())
diff --git a/Lib/lib2to3/fixes/fix_import.py b/Lib/lib2to3/fixes/fix_import.py
index 58e914913f..4c751337dc 100644
--- a/Lib/lib2to3/fixes/fix_import.py
+++ b/Lib/lib2to3/fixes/fix_import.py
@@ -13,55 +13,78 @@ Becomes:
 # Local imports
 from .. import fixer_base
 from os.path import dirname, join, exists, pathsep
-from ..fixer_util import FromImport, syms
+from ..fixer_util import FromImport, syms, token
+
+
+def traverse_imports(names):
+    """
+    Walks over all the names imported in a dotted_as_names node.
+    """
+    pending = [names]
+    while pending:
+        node = pending.pop()
+        if node.type == token.NAME:
+            yield node.value
+        elif node.type == syms.dotted_name:
+            yield "".join([ch.value for ch in node.children])
+        elif node.type == syms.dotted_as_name:
+            pending.append(node.children[0])
+        elif node.type == syms.dotted_as_names:
+            pending.extend(node.children[::-2])
+        else:
+            raise AssertionError("unkown node type")
+
 
 class FixImport(fixer_base.BaseFix):
 
     PATTERN = """
-    import_from< type='from' imp=any 'import' ['('] any [')'] >
+    import_from< 'from' imp=any 'import' ['('] any [')'] >
     |
-    import_name< type='import' imp=any >
+    import_name< 'import' imp=any >
     """
 
     def transform(self, node, results):
         imp = results['imp']
 
-        mod_name = unicode(imp.children[0] if imp.type == syms.dotted_as_name \
-                               else imp)
-
-        if mod_name.startswith('.'):
-            # Already a new-style import
-            return
-
-        if not probably_a_local_import(mod_name, self.filename):
-            # I guess this is a global import -- skip it!
-            return
-
-        if results['type'].value == 'from':
+        if node.type == syms.import_from:
             # Some imps are top-level (eg: 'import ham')
             # some are first level (eg: 'import ham.eggs')
             # some are third level (eg: 'import ham.eggs as spam')
             # Hence, the loop
             while not hasattr(imp, 'value'):
                 imp = imp.children[0]
-            imp.value = "." + imp.value
-            node.changed()
+            if self.probably_a_local_import(imp.value):
+                imp.value = "." + imp.value
+                imp.changed()
+                return node
         else:
-            new = FromImport('.', getattr(imp, 'content', None) or [imp])
+            have_local = False
+            have_absolute = False
+            for mod_name in traverse_imports(imp):
+                if self.probably_a_local_import(mod_name):
+                    have_local = True
+                else:
+                    have_absolute = True
+            if have_absolute:
+                if have_local:
+                    # We won't handle both sibling and absolute imports in the
+                    # same statement at the moment.
+                    self.warning(node, "absolute and local imports together")
+                return
+
+            new = FromImport('.', [imp])
             new.set_prefix(node.get_prefix())
-            node = new
-        return node
+            return new
 
-def probably_a_local_import(imp_name, file_path):
-    # Must be stripped because the right space is included by the parser
-    imp_name = imp_name.split('.', 1)[0].strip()
-    base_path = dirname(file_path)
-    base_path = join(base_path, imp_name)
-    # If there is no __init__.py next to the file its not in a package
-    # so can't be a relative import.
-    if not exists(join(dirname(base_path), '__init__.py')):
+    def probably_a_local_import(self, imp_name):
+        imp_name = imp_name.split('.', 1)[0]
+        base_path = dirname(self.filename)
+        base_path = join(base_path, imp_name)
+        # If there is no __init__.py next to the file its not in a package
+        # so can't be a relative import.
+        if not exists(join(dirname(base_path), '__init__.py')):
+            return False
+        for ext in ['.py', pathsep, '.pyc', '.so', '.sl', '.pyd']:
+            if exists(base_path + ext):
+                return True
         return False
-    for ext in ['.py', pathsep, '.pyc', '.so', '.sl', '.pyd']:
-        if exists(base_path + ext):
-            return True
-    return False
diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py
index 75770c915f..98d440698c 100644
--- a/Lib/lib2to3/fixes/fix_imports.py
+++ b/Lib/lib2to3/fixes/fix_imports.py
@@ -118,7 +118,7 @@ class FixImports(fixer_base.BaseFix):
     def transform(self, node, results):
         import_mod = results.get("module_name")
         if import_mod:
-            new_name = self.mapping[(import_mod or mod_name).value]
+            new_name = self.mapping[import_mod.value]
             import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
             if "name_import" in results:
                 # If it's not a "from x import x, y" or "import x as y" import,
@@ -129,10 +129,8 @@ class FixImports(fixer_base.BaseFix):
                 # line (e.g., "import StringIO, urlparse"). The problem is that I
                 # can't figure out an easy way to make a pattern recognize the
                 # keys of MAPPING randomly sprinkled in an import statement.
-                while True:
-                    results = self.match(node)
-                    if not results:
-                        break
+                results = self.match(node)
+                if results:
                     self.transform(node, results)
         else:
             # Replace usage of the module.
diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py
index 5fd6af5514..873ecf1b9e 100644
--- a/Lib/lib2to3/fixes/fix_long.py
+++ b/Lib/lib2to3/fixes/fix_long.py
@@ -5,20 +5,18 @@
 """
 
 # Local imports
-from .. import pytree
 from .. import fixer_base
-from ..fixer_util import Name, Number
+from ..fixer_util import Name, Number, is_probably_builtin
 
 
 class FixLong(fixer_base.BaseFix):
 
     PATTERN = "'long'"
 
-    static_long = Name("long")
     static_int = Name("int")
 
     def transform(self, node, results):
-        assert node == self.static_long, node
-        new = self.static_int.clone()
-        new.set_prefix(node.get_prefix())
-        return new
+        if is_probably_builtin(node):
+            new = self.static_int.clone()
+            new.set_prefix(node.get_prefix())
+            return new