diff options
author | Allan Sandfeld Jensen <allan.jensen@theqtcompany.com> | 2016-05-09 14:22:11 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2016-05-09 15:11:45 +0000 |
commit | 2ddb2d3e14eef3de7dbd0cef553d669b9ac2361c (patch) | |
tree | e75f511546c5fd1a173e87c1f9fb11d7ac8d1af3 /chromium/tools/checkperms/checkperms.py | |
parent | a4f3d46271c57e8155ba912df46a05559d14726e (diff) | |
download | qtwebengine-chromium-2ddb2d3e14eef3de7dbd0cef553d669b9ac2361c.tar.gz |
BASELINE: Update Chromium to 51.0.2704.41
Also adds in all smaller components by reversing logic for exclusion.
Change-Id: Ibf90b506e7da088ea2f65dcf23f2b0992c504422
Reviewed-by: Joerg Bornemann <joerg.bornemann@theqtcompany.com>
Diffstat (limited to 'chromium/tools/checkperms/checkperms.py')
-rwxr-xr-x | chromium/tools/checkperms/checkperms.py | 486 |
1 files changed, 486 insertions, 0 deletions
diff --git a/chromium/tools/checkperms/checkperms.py b/chromium/tools/checkperms/checkperms.py new file mode 100755 index 00000000000..81b0f4f0ba9 --- /dev/null +++ b/chromium/tools/checkperms/checkperms.py @@ -0,0 +1,486 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Makes sure files have the right permissions. + +Some developers have broken SCM configurations that flip the executable +permission on for no good reason. Unix developers who run ls --color will then +see .cc files in green and get confused. + +- For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS. +- For file extensions that must not be executable, add it to + NOT_EXECUTABLE_EXTENSIONS. +- To ignore all the files inside a directory, add it to IGNORED_PATHS. +- For file base name with ambiguous state and that should not be checked for + shebang, add it to IGNORED_FILENAMES. + +Any file not matching the above will be opened and looked if it has a shebang +or an ELF header. If this does not match the executable bit on the file, the +file will be flagged. + +Note that all directory separators must be slashes (Unix-style) and not +backslashes. All directories should be relative to the source root and all +file paths should be only lowercase. +""" + +import json +import logging +import optparse +import os +import stat +import string +import subprocess +import sys + +#### USER EDITABLE SECTION STARTS HERE #### + +# Files with these extensions must have executable bit set. +# +# Case-sensitive. +EXECUTABLE_EXTENSIONS = ( + 'bat', + 'dll', + 'exe', +) + +# Files for which the executable bit may or may not be set. +IGNORED_EXTENSIONS = ( + 'dylib', +) + +# These files must have executable bit set. +# +# Case-insensitive, lower-case only. +EXECUTABLE_PATHS = ( + 'chrome/test/data/app_shim/app_shim_32_bit.app/contents/' + 'macos/app_mode_loader', + 'chrome/test/data/extensions/uitest/plugins/plugin.plugin/contents/' + 'macos/testnetscapeplugin', + 'chrome/test/data/extensions/uitest/plugins_private/plugin.plugin/contents/' + 'macos/testnetscapeplugin', +) + +# These files must not have the executable bit set. This is mainly a performance +# optimization as these files are not checked for shebang. The list was +# partially generated from: +# git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g +# +# Case-sensitive. +NON_EXECUTABLE_EXTENSIONS = ( + '1', + '3ds', + 'S', + 'am', + 'applescript', + 'asm', + 'c', + 'cc', + 'cfg', + 'chromium', + 'cpp', + 'crx', + 'cs', + 'css', + 'cur', + 'def', + 'der', + 'expected', + 'gif', + 'grd', + 'gyp', + 'gypi', + 'h', + 'hh', + 'htm', + 'html', + 'hyph', + 'ico', + 'idl', + 'java', + 'jpg', + 'js', + 'json', + 'm', + 'm4', + 'mm', + 'mms', + 'mock-http-headers', + 'nexe', + 'nmf', + 'onc', + 'pat', + 'patch', + 'pdf', + 'pem', + 'plist', + 'png', + 'proto', + 'rc', + 'rfx', + 'rgs', + 'rules', + 'spec', + 'sql', + 'srpc', + 'svg', + 'tcl', + 'test', + 'tga', + 'txt', + 'vcproj', + 'vsprops', + 'webm', + 'word', + 'xib', + 'xml', + 'xtb', + 'zip', +) + +# These files must not have executable bit set. +# +# Case-insensitive, lower-case only. +NON_EXECUTABLE_PATHS = ( + 'build/android/tests/symbolize/liba.so', + 'build/android/tests/symbolize/libb.so', + 'chrome/installer/mac/sign_app.sh.in', + 'chrome/installer/mac/sign_versioned_dir.sh.in', + 'chrome/test/data/extensions/uitest/plugins/plugin32.so', + 'chrome/test/data/extensions/uitest/plugins/plugin64.so', + 'chrome/test/data/extensions/uitest/plugins_private/plugin32.so', + 'chrome/test/data/extensions/uitest/plugins_private/plugin64.so', + 'courgette/testdata/elf-32-1', + 'courgette/testdata/elf-32-2', + 'courgette/testdata/elf-64', +) + +# File names that are always whitelisted. (These are mostly autoconf spew.) +# +# Case-sensitive. +IGNORED_FILENAMES = ( + 'config.guess', + 'config.sub', + 'configure', + 'depcomp', + 'install-sh', + 'missing', + 'mkinstalldirs', + 'naclsdk', + 'scons', +) + +# File paths starting with one of these will be ignored as well. +# Please consider fixing your file permissions, rather than adding to this list. +# +# Case-insensitive, lower-case only. +IGNORED_PATHS = ( + 'base/third_party/libevent/autogen.sh', + 'base/third_party/libevent/test/test.sh', + 'native_client_sdk/src/build_tools/sdk_tools/third_party/fancy_urllib/' + '__init__.py', + 'out/', + # TODO(maruel): Fix these. + 'third_party/bintrees/', + 'third_party/closure_linter/', + 'third_party/devscripts/licensecheck.pl.vanilla', + 'third_party/hyphen/', + 'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl', + 'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh', + 'third_party/lcov/contrib/galaxy/conglomerate_functions.pl', + 'third_party/lcov/contrib/galaxy/gen_makefile.sh', + 'third_party/libxml/linux/xml2-config', + 'third_party/libxml/src/ltmain.sh', + 'third_party/mesa/', + 'third_party/protobuf/', + 'third_party/python_gflags/gflags.py', + 'third_party/sqlite/', + 'third_party/talloc/script/mksyms.sh', + 'third_party/tcmalloc/', + 'third_party/tlslite/setup.py', + # TODO(nednguyen): Remove this when telemetry is moved to catapult + 'tools/telemetry/third_party/', +) + +#### USER EDITABLE SECTION ENDS HERE #### + +assert (set(EXECUTABLE_EXTENSIONS) & set(IGNORED_EXTENSIONS) & + set(NON_EXECUTABLE_EXTENSIONS) == set()) +assert set(EXECUTABLE_PATHS) & set(NON_EXECUTABLE_PATHS) == set() + +VALID_CHARS = set(string.ascii_lowercase + string.digits + '/-_.') +for paths in (EXECUTABLE_PATHS, NON_EXECUTABLE_PATHS, IGNORED_PATHS): + assert all([set(path).issubset(VALID_CHARS) for path in paths]) + + +def capture(cmd, cwd): + """Returns the output of a command. + + Ignores the error code or stderr. + """ + logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd)) + env = os.environ.copy() + env['LANGUAGE'] = 'en_US.UTF-8' + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) + return p.communicate()[0] + + +def get_git_root(dir_path): + """Returns the git checkout root or None.""" + root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip() + if root: + return root + + +def is_ignored(rel_path): + """Returns True if rel_path is in our whitelist of files to ignore.""" + rel_path = rel_path.lower() + return ( + os.path.basename(rel_path) in IGNORED_FILENAMES or + rel_path.lower().startswith(IGNORED_PATHS)) + + +def must_be_executable(rel_path): + """The file name represents a file type that must have the executable bit + set. + """ + return (os.path.splitext(rel_path)[1][1:] in EXECUTABLE_EXTENSIONS or + rel_path.lower() in EXECUTABLE_PATHS) + + +def ignored_extension(rel_path): + """The file name represents a file type that may or may not have the + executable set. + """ + return os.path.splitext(rel_path)[1][1:] in IGNORED_EXTENSIONS + + +def must_not_be_executable(rel_path): + """The file name represents a file type that must not have the executable + bit set. + """ + return (os.path.splitext(rel_path)[1][1:] in NON_EXECUTABLE_EXTENSIONS or + rel_path.lower() in NON_EXECUTABLE_PATHS) + + +def has_executable_bit(full_path): + """Returns if any executable bit is set.""" + permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + return bool(permission & os.stat(full_path).st_mode) + + +def has_shebang_or_is_elf(full_path): + """Returns if the file starts with #!/ or is an ELF binary. + + full_path is the absolute path to the file. + """ + with open(full_path, 'rb') as f: + data = f.read(4) + return (data[:3] == '#!/' or data == '#! /', data == '\x7fELF') + + +def check_file(root_path, rel_path): + """Checks the permissions of the file whose path is root_path + rel_path and + returns an error if it is inconsistent. Returns None on success. + + It is assumed that the file is not ignored by is_ignored(). + + If the file name is matched with must_be_executable() or + must_not_be_executable(), only its executable bit is checked. + Otherwise, the first few bytes of the file are read to verify if it has a + shebang or ELF header and compares this with the executable bit on the file. + """ + full_path = os.path.join(root_path, rel_path) + def result_dict(error): + return { + 'error': error, + 'full_path': full_path, + 'rel_path': rel_path, + } + try: + bit = has_executable_bit(full_path) + except OSError: + # It's faster to catch exception than call os.path.islink(). The Chromium + # tree may have invalid symlinks. + return None + + if must_be_executable(rel_path): + if not bit: + return result_dict('Must have executable bit set') + return + if must_not_be_executable(rel_path): + if bit: + return result_dict('Must not have executable bit set') + return + if ignored_extension(rel_path): + return + + # For the others, it depends on the file header. + (shebang, elf) = has_shebang_or_is_elf(full_path) + if bit != (shebang or elf): + if bit: + return result_dict('Has executable bit but not shebang or ELF header') + if shebang: + return result_dict('Has shebang but not executable bit') + return result_dict('Has ELF header but not executable bit') + + +def check_files(root, files): + gen = (check_file(root, f) for f in files if not is_ignored(f)) + return filter(None, gen) + + +class ApiBase(object): + def __init__(self, root_dir, bare_output): + self.root_dir = root_dir + self.bare_output = bare_output + self.count = 0 + self.count_read_header = 0 + + def check_file(self, rel_path): + logging.debug('check_file(%s)' % rel_path) + self.count += 1 + + if (not must_be_executable(rel_path) and + not must_not_be_executable(rel_path)): + self.count_read_header += 1 + + return check_file(self.root_dir, rel_path) + + def check_dir(self, rel_path): + return self.check(rel_path) + + def check(self, start_dir): + """Check the files in start_dir, recursively check its subdirectories.""" + errors = [] + items = self.list_dir(start_dir) + logging.info('check(%s) -> %d' % (start_dir, len(items))) + for item in items: + full_path = os.path.join(self.root_dir, start_dir, item) + rel_path = full_path[len(self.root_dir) + 1:] + if is_ignored(rel_path): + continue + if os.path.isdir(full_path): + # Depth first. + errors.extend(self.check_dir(rel_path)) + else: + error = self.check_file(rel_path) + if error: + errors.append(error) + return errors + + def list_dir(self, start_dir): + """Lists all the files and directory inside start_dir.""" + return sorted( + x for x in os.listdir(os.path.join(self.root_dir, start_dir)) + if not x.startswith('.') + ) + + +class ApiAllFilesAtOnceBase(ApiBase): + _files = None + + def list_dir(self, start_dir): + """Lists all the files and directory inside start_dir.""" + if self._files is None: + self._files = sorted(self._get_all_files()) + if not self.bare_output: + print 'Found %s files' % len(self._files) + start_dir = start_dir[len(self.root_dir) + 1:] + return [ + x[len(start_dir):] for x in self._files if x.startswith(start_dir) + ] + + def _get_all_files(self): + """Lists all the files and directory inside self._root_dir.""" + raise NotImplementedError() + + +class ApiGit(ApiAllFilesAtOnceBase): + def _get_all_files(self): + return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines() + + +def get_scm(dir_path, bare): + """Returns a properly configured ApiBase instance.""" + cwd = os.getcwd() + root = get_git_root(dir_path or cwd) + if root: + if not bare: + print('Found git repository at %s' % root) + return ApiGit(dir_path or root, bare) + + # Returns a non-scm aware checker. + if not bare: + print('Failed to determine the SCM for %s' % dir_path) + return ApiBase(dir_path or cwd, bare) + + +def main(): + usage = """Usage: python %prog [--root <root>] [tocheck] + tocheck Specifies the directory, relative to root, to check. This defaults + to "." so it checks everything. + +Examples: + python %prog + python %prog --root /path/to/source chrome""" + + parser = optparse.OptionParser(usage=usage) + parser.add_option( + '--root', + help='Specifies the repository root. This defaults ' + 'to the checkout repository root') + parser.add_option( + '-v', '--verbose', action='count', default=0, help='Print debug logging') + parser.add_option( + '--bare', + action='store_true', + default=False, + help='Prints the bare filename triggering the checks') + parser.add_option( + '--file', action='append', dest='files', + help='Specifics a list of files to check the permissions of. Only these ' + 'files will be checked') + parser.add_option('--json', help='Path to JSON output file') + options, args = parser.parse_args() + + levels = [logging.ERROR, logging.INFO, logging.DEBUG] + logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)]) + + if len(args) > 1: + parser.error('Too many arguments used') + + if options.root: + options.root = os.path.abspath(options.root) + + if options.files: + errors = check_files(options.root, options.files) + else: + api = get_scm(options.root, options.bare) + start_dir = args[0] if args else api.root_dir + errors = api.check(start_dir) + + if not options.bare: + print('Processed %s files, %d files where tested for shebang/ELF ' + 'header' % (api.count, api.count_read_header)) + + if options.json: + with open(options.json, 'w') as f: + json.dump(errors, f) + + if errors: + if options.bare: + print '\n'.join(e['full_path'] for e in errors) + else: + print '\nFAILED\n' + print '\n'.join('%s: %s' % (e['full_path'], e['error']) for e in errors) + return 1 + if not options.bare: + print '\nSUCCESS\n' + return 0 + + +if '__main__' == __name__: + sys.exit(main()) |