merge with lehmannro/sphinx-i18n

3 files changed, 153 insertions, 85 deletions

diff --git a/utils/check_sources.py b/utils/check_sources.py
index 0571ab1e..c412742b 100755
--- a/utils/check_sources.py
+++ b/utils/check_sources.py
@@ -12,10 +12,16 @@
 """
 
 import sys, os, re
-import getopt
 import cStringIO
+from optparse import OptionParser
 from os.path import join, splitext, abspath
 
+if sys.version_info >= (3, 0):
+    def b(s):
+        return s.encode('utf-8')
+else:
+    b = str
+
 
 checkers = {}
 
@@ -30,26 +36,26 @@ def checker(*suffixes, **kwds):
 
 
 name_mail_re = r'[\w ]+(<.*?>)?'
-copyright_re = re.compile(r'^    :copyright: Copyright 200\d(-20\d\d)? '
-                          r'by %s(, %s)*[,.]$' %
-                          (name_mail_re, name_mail_re))
-license_re = re.compile(r"    :license: (.*?).\n")
-copyright_2_re = re.compile(r'^                %s(, %s)*[,.]$' %
-                            (name_mail_re, name_mail_re))
-coding_re    = re.compile(r'coding[:=]\s*([-\w.]+)')
-not_ix_re    = re.compile(r'\bnot\s+\S+?\s+i[sn]\s\S+')
-is_const_re  = re.compile(r'if.*?==\s+(None|False|True)\b')
-
-misspellings = ["developement", "adress", "verificate",  # ALLOW-MISSPELLING
-                "informations"]                          # ALLOW-MISSPELLING
-
-
-@checker('.py')
-def check_syntax(fn, lines):
-    try:
-        compile(''.join(lines), fn, "exec")
-    except SyntaxError, err:
-        yield 0, "not compilable: %s" % err
+copyright_re = re.compile(b(r'^    :copyright: Copyright 200\d(-20\d\d)? '
+                            r'by %s(, %s)*[,.]$' %
+                            (name_mail_re, name_mail_re)))
+license_re = re.compile(b(r"    :license: (.*?).\n"))
+copyright_2_re = re.compile(b(r'^                %s(, %s)*[,.]$' %
+                             (name_mail_re, name_mail_re)))
+coding_re    = re.compile(b(r'coding[:=]\s*([-\w.]+)'))
+not_ix_re    = re.compile(b(r'\bnot\s+\S+?\s+i[sn]\s\S+'))
+is_const_re  = re.compile(b(r'if.*?==\s+(None|False|True)\b'))
+
+misspellings = [b("developement"), b("adress"), # ALLOW-MISSPELLING
+                b("verificate"), b("informations")] # ALLOW-MISSPELLING
+
+if sys.version_info < (3, 0):
+    @checker('.py')
+    def check_syntax(fn, lines):
+        try:
+            compile(b('').join(lines), fn, "exec")
+        except SyntaxError, err:
+            yield 0, "not compilable: %s" % err
 
 
 @checker('.py')
@@ -61,8 +67,8 @@ def check_style_and_encoding(fn, lines):
         if lno < 2:
             co = coding_re.search(line)
             if co:
-                encoding = co.group(1)
-        if line.strip().startswith('#'):
+                encoding = co.group(1).decode('ascii')
+        if line.strip().startswith(b('#')):
             continue
         #m = not_ix_re.search(line)
         #if m:
@@ -82,7 +88,7 @@ def check_style_and_encoding(fn, lines):
 def check_fileheader(fn, lines):
     # line number correction
     c = 1
-    if lines[0:1] == ['#!/usr/bin/env python\n']:
+    if lines[0:1] == [b('#!/usr/bin/env python\n')]:
         lines = lines[1:]
         c = 2
 
@@ -91,38 +97,38 @@ def check_fileheader(fn, lines):
     for lno, l in enumerate(lines):
         llist.append(l)
         if lno == 0:
-            if l == '# -*- coding: rot13 -*-\n':
+            if l == b('# -*- coding: rot13 -*-\n'):
                 # special-case pony package
                 return
-            elif l != '# -*- coding: utf-8 -*-\n':
+            elif l != b('# -*- coding: utf-8 -*-\n'):
                 yield 1, "missing coding declaration"
         elif lno == 1:
-            if l != '"""\n' and l != 'r"""\n':
+            if l != b('"""\n') and l != b('r"""\n'):
                 yield 2, 'missing docstring begin (""")'
             else:
                 docopen = True
         elif docopen:
-            if l == '"""\n':
+            if l == b('"""\n'):
                 # end of docstring
                 if lno <= 4:
                     yield lno+c, "missing module name in docstring"
                 break
 
-            if l != "\n" and l[:4] != '    ' and docopen:
+            if l != b("\n") and l[:4] != b('    ') and docopen:
                 yield lno+c, "missing correct docstring indentation"
 
             if lno == 2:
                 # if not in package, don't check the module name
                 modname = fn[:-3].replace('/', '.').replace('.__init__', '')
                 while modname:
-                    if l.lower()[4:-1] == modname:
+                    if l.lower()[4:-1] == b(modname):
                         break
                     modname = '.'.join(modname.split('.')[1:])
                 else:
                     yield 3, "wrong module name in docstring heading"
                 modnamelen = len(l.strip())
             elif lno == 3:
-                if l.strip() != modnamelen * "~":
+                if l.strip() != modnamelen * b("~"):
                     yield 4, "wrong module name underline, should be ~~~...~"
 
     else:
@@ -145,16 +151,16 @@ def check_fileheader(fn, lines):
 @checker('.py', '.html', '.rst')
 def check_whitespace_and_spelling(fn, lines):
     for lno, line in enumerate(lines):
-        if "\t" in line:
+        if b("\t") in line:
             yield lno+1, "OMG TABS!!!1 "
-        if line[:-1].rstrip(' \t') != line[:-1]:
+        if line[:-1].rstrip(b(' \t')) != line[:-1]:
             yield lno+1, "trailing whitespace"
         for word in misspellings:
-            if word in line and 'ALLOW-MISSPELLING' not in line:
+            if word in line and b('ALLOW-MISSPELLING') not in line:
                 yield lno+1, '"%s" used' % word
 
 
-bad_tags = ('<u>', '<s>', '<strike>', '<center>', '<font')
+bad_tags = map(b, ['<u>', '<s>', '<strike>', '<center>', '<font'])
 
 @checker('.html')
 def check_xhtml(fn, lines):
@@ -165,34 +171,32 @@ def check_xhtml(fn, lines):
 
 
 def main(argv):
-    try:
-        gopts, args = getopt.getopt(argv[1:], "vi:")
-    except getopt.GetoptError:
-        print "Usage: %s [-v] [-i ignorepath]* [path]" % argv[0]
-        return 2
-    opts = {}
-    for opt, val in gopts:
-        if opt == '-i':
-            val = abspath(val)
-        opts.setdefault(opt, []).append(val)
+    parser = OptionParser(usage='Usage: %prog [-v] [-i ignorepath]* [path]')
+    parser.add_option('-v', '--verbose', dest='verbose', default=False,
+                      action='store_true')
+    parser.add_option('-i', '--ignore-path', dest='ignored_paths',
+                      default=[], action='append')
+    options, args = parser.parse_args(argv[1:])
 
     if len(args) == 0:
         path = '.'
     elif len(args) == 1:
         path = args[0]
     else:
-        print "Usage: %s [-v] [-i ignorepath]* [path]" % argv[0]
-        return 2
+        print args
+        parser.error('No more then one path supported')
 
-    verbose = '-v' in opts
+    verbose = options.verbose
+    ignored_paths = set(abspath(p) for p in options.ignored_paths)
 
     num = 0
     out = cStringIO.StringIO()
 
     for root, dirs, files in os.walk(path):
-        if '.svn' in dirs:
-            dirs.remove('.svn')
-        if '-i' in opts and abspath(root) in opts['-i']:
+        for vcs_dir in ['.svn', '.hg', '.git']:
+            if vcs_dir in dirs:
+                dirs.remove(vcs_dir)
+        if abspath(root) in ignored_paths:
             del dirs[:]
             continue
         in_check_pkg = root.startswith('./sphinx')
@@ -201,7 +205,7 @@ def main(argv):
             fn = join(root, fn)
             if fn[:2] == './': fn = fn[2:]
 
-            if '-i' in opts and abspath(fn) in opts['-i']:
+            if abspath(fn) in ignored_paths:
                 continue
 
             ext = splitext(fn)[1]
@@ -213,8 +217,11 @@ def main(argv):
                 print "Checking %s..." % fn
 
             try:
-                f = open(fn, 'r')
-                lines = list(f)
+                f = open(fn, 'rb')
+                try:
+                    lines = list(f)
+                finally:
+                    f.close()
             except (IOError, OSError), err:
                 print "%s: cannot open: %s" % (fn, err)
                 num += 1
diff --git a/utils/convert.py b/utils/convert.py
new file mode 100644
index 00000000..f025c49a
--- /dev/null
+++ b/utils/convert.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+    Converts files with 2to3
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Creates a Python 3 version of each file.
+
+    The Python3 version of a file foo.py will be called foo3.py.
+
+    :copyright: Copyright 2010 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import os
+import sys
+from glob import iglob
+from optparse import OptionParser
+from shutil import copy
+from distutils.util import run_2to3
+
+def main(argv):
+    parser = OptionParser(usage='%prog [path]')
+    parser.add_option('-i', '--ignorepath', dest='ignored_paths',
+                      action='append', default=[])
+    options, args = parser.parse_args(argv)
+
+    ignored_paths = {os.path.abspath(p) for p in options.ignored_paths}
+
+    path = os.path.abspath(args[0]) if args else os.getcwd()
+    convertables = []
+    for filename in iglob(os.path.join(path, '*.py')):
+        if filename in ignored_paths:
+            continue
+        basename, ext = os.path.splitext(filename)
+        if basename.endswith('3'):
+            continue
+        filename3 = basename + '3' + ext
+        copy(filename, filename3)
+        convertables.append(filename3)
+    run_2to3(convertables)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/utils/reindent.py b/utils/reindent.py
index c499f671..59828fd8 100755
--- a/utils/reindent.py
+++ b/utils/reindent.py
@@ -1,16 +1,14 @@
 #! /usr/bin/env python
 
 # Released to the public domain, by Tim Peters, 03 October 2000.
-# -B option added by Georg Brandl, 2006.
 
 """reindent [-d][-r][-v] [ path ... ]
 
--d (--dryrun)  Dry run.  Analyze, but don't make any changes to files.
--r (--recurse) Recurse.  Search for all .py files in subdirectories too.
--B (--no-backup)         Don't write .bak backup files.
--v (--verbose) Verbose.  Print informative msgs; else only names of \
-changed files.
--h (--help)    Help.     Print this usage information and exit.
+-d (--dryrun)   Dry run.   Analyze, but don't make any changes to, files.
+-r (--recurse)  Recurse.   Search for all .py files in subdirectories too.
+-n (--nobackup) No backup. Does not make a ".bak" file before reindenting.
+-v (--verbose)  Verbose.   Print informative msgs; else no output.
+-h (--help)     Help.      Print this usage information and exit.
 
 Change Python (.py) files to use 4-space indents and no hard tab characters.
 Also trim excess spaces and tabs from ends of lines, and remove empty lines
@@ -34,18 +32,30 @@ resulting .py file won't change it again).
 The hard part of reindenting is figuring out what to do with comment
 lines.  So long as the input files get a clean bill of health from
 tabnanny.py, reindent should do a good job.
+
+The backup file is a copy of the one that is being reindented. The ".bak"
+file is generated with shutil.copy(), but some corner cases regarding
+user/group and permissions could leave the backup file more readable that
+you'd prefer. You can always use the --nobackup option to prevent this.
 """
 
 __version__ = "1"
 
 import tokenize
-import os
+import os, shutil
 import sys
 
-verbose = 0
-recurse = 0
-dryrun  = 0
-no_backup = 0
+if sys.version_info >= (3, 0):
+    def tokens(readline, tokeneater):
+        for token in tokenize.tokenize(readline):
+            yield tokeneater(*token)
+else:
+    tokens = tokenize.tokenize
+
+verbose    = 0
+recurse    = 0
+dryrun     = 0
+makebackup = True
 
 def usage(msg=None):
     if msg is not None:
@@ -61,12 +71,10 @@ def errprint(*args):
 
 def main():
     import getopt
-    global verbose, recurse, dryrun, no_backup
-
+    global verbose, recurse, dryrun, makebackup
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "drvhB",
-                                   ["dryrun", "recurse", "verbose", "help",
-                                    "no-backup"])
+        opts, args = getopt.getopt(sys.argv[1:], "drnvh",
+                        ["dryrun", "recurse", "nobackup", "verbose", "help"])
     except getopt.error, msg:
         usage(msg)
         return
@@ -75,10 +83,10 @@ def main():
             dryrun += 1
         elif o in ('-r', '--recurse'):
             recurse += 1
+        elif o in ('-n', '--nobackup'):
+            makebackup = False
         elif o in ('-v', '--verbose'):
             verbose += 1
-        elif o in ('-B', '--no-backup'):
-            no_backup += 1
         elif o in ('-h', '--help'):
             usage()
             return
@@ -98,7 +106,8 @@ def check(file):
         for name in names:
             fullname = os.path.join(file, name)
             if ((recurse and os.path.isdir(fullname) and
-                 not os.path.islink(fullname))
+                 not os.path.islink(fullname) and
+                 not os.path.split(fullname)[1].startswith("."))
                 or name.lower().endswith(".py")):
                 check(fullname)
         return
@@ -118,26 +127,35 @@ def check(file):
             print "changed."
             if dryrun:
                 print "But this is a dry run, so leaving it alone."
-        else:
-            print "reindented", file, \
-                  (dryrun and "(dry run => not really)" or "")
         if not dryrun:
-            if not no_backup:
-                bak = file + ".bak"
-                if os.path.exists(bak):
-                    os.remove(bak)
-                os.rename(file, bak)
+            bak = file + ".bak"
+            if makebackup:
+                shutil.copyfile(file, bak)
                 if verbose:
-                    print "renamed", file, "to", bak
+                    print "backed up", file, "to", bak
             f = open(file, "w")
             r.write(f)
             f.close()
             if verbose:
                 print "wrote new", file
+        return True
     else:
         if verbose:
             print "unchanged."
+        return False
+
+def _rstrip(line, JUNK='\n \t'):
+    """Return line stripped of trailing spaces, tabs, newlines.
+
+    Note that line.rstrip() instead also strips sundry control characters,
+    but at least one known Emacs user expects to keep junk like that, not
+    mentioning Barry by name or anything <wink>.
+    """
 
+    i = len(line)
+    while i > 0 and line[i-1] in JUNK:
+        i -= 1
+    return line[:i]
 
 class Reindenter:
 
@@ -151,7 +169,7 @@ class Reindenter:
         # File lines, rstripped & tab-expanded.  Dummy at start is so
         # that we can use tokenize's 1-based line numbering easily.
         # Note that a line is all-blank iff it's "\n".
-        self.lines = [line.rstrip('\n \t').expandtabs() + "\n"
+        self.lines = [_rstrip(line).expandtabs() + "\n"
                       for line in self.raw]
         self.lines.insert(0, None)
         self.index = 1  # index into self.lines of next line
@@ -163,7 +181,7 @@ class Reindenter:
         self.stats = []
 
     def run(self):
-        tokenize.tokenize(self.getline, self.tokeneater)
+        tokens(self.getline, self.tokeneater)
         # Remove trailing empty lines.
         lines = self.lines
         while lines and lines[-1] == "\n":