diff options
Diffstat (limited to 'chromium/tools/clang/scripts/package.py')
| -rwxr-xr-x | chromium/tools/clang/scripts/package.py | 335 |
1 files changed, 160 insertions, 175 deletions
diff --git a/chromium/tools/clang/scripts/package.py b/chromium/tools/clang/scripts/package.py index 8310d7caa3b..0af8d1666bc 100755 --- a/chromium/tools/clang/scripts/package.py +++ b/chromium/tools/clang/scripts/package.py @@ -15,6 +15,8 @@ import subprocess import sys import tarfile +from update import RELEASE_VERSION + # Path constants. THIS_DIR = os.path.dirname(__file__) CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) @@ -87,20 +89,11 @@ def RunGsutil(args): return subprocess.call([sys.executable, GetGsutilPath()] + args) -def GsutilArchiveExists(archive_name, platform): - gsutil_args = ['-q', 'stat', - 'gs://chromium-browser-clang-staging/%s/%s.tgz' % - (platform, archive_name)] - return RunGsutil(gsutil_args) == 0 - - -def MaybeUpload(args, archive_name, platform): - gsutil_args = ['cp', '-a', 'public-read', - '%s.tgz' % archive_name, - 'gs://chromium-browser-clang-staging/%s/%s.tgz' % - (platform, archive_name)] - if args.upload: - print 'Uploading %s to Google Cloud Storage...' % archive_name +def MaybeUpload(do_upload, filename, platform, extra_gsutil_args=[]): + gsutil_args = ['cp'] + extra_gsutil_args + ['-a', 'public-read', filename, + 'gs://chromium-browser-clang-staging/%s/%s' % (platform, filename)] + if do_upload: + print 'Uploading %s to Google Cloud Storage...' % filename exit_code = RunGsutil(gsutil_args) if exit_code != 0: print "gsutil failed, exit_code: %s" % exit_code @@ -184,28 +177,15 @@ def main(): platform = 'Linux_x64' with open('buildlog.txt', 'w') as log: - Tee('Diff in llvm:\n', log) - TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False) - TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False) - Tee('Diff in llvm/tools/clang:\n', log) - TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')], - log, fail_hard=False) - TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')], - log, fail_hard=False) - # TODO(thakis): compiler-rt is in projects/compiler-rt on Windows but - # llvm/compiler-rt elsewhere. So this diff call is currently only right on - # Windows. - Tee('Diff in llvm/compiler-rt:\n', log) - TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')], - log, fail_hard=False) - TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')], - log, fail_hard=False) - Tee('Diff in llvm/projects/libcxx:\n', log) - TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxx')], - log, fail_hard=False) - TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxx')], - log, fail_hard=False) - + if os.path.exists(LLVM_DIR): + Tee('Diff in llvm:\n', log) + cwd = os.getcwd() + os.chdir(LLVM_DIR) + TeeCmd(['git', 'status'], log, fail_hard=False) + TeeCmd(['git', 'diff'], log, fail_hard=False) + os.chdir(cwd) + else: + Tee('No previous llvm checkout.\n', log) Tee('Starting build\n', log) # Do a clobber build. @@ -216,8 +196,8 @@ def main(): opt_flags = [] if sys.platform.startswith('linux'): opt_flags += ['--lto-lld'] - build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), - '--bootstrap', '--disable-asserts', '--force-local-build', + build_cmd = [sys.executable, os.path.join(THIS_DIR, 'build.py'), + '--bootstrap', '--disable-asserts', '--run-tests'] + opt_flags TeeCmd(build_cmd, log) @@ -230,149 +210,152 @@ def main(): # Copy a whitelist of files to the directory we're going to tar up. # This supports the same patterns that the fnmatch module understands. + # '$V' is replaced by RELEASE_VERSION further down. exe_ext = '.exe' if sys.platform == 'win32' else '' - want = ['bin/llvm-pdbutil' + exe_ext, - 'bin/llvm-symbolizer' + exe_ext, - 'bin/llvm-undname' + exe_ext, - # Copy built-in headers (lib/clang/3.x.y/include). - 'lib/clang/*/include/*', - 'lib/clang/*/share/asan_blacklist.txt', - 'lib/clang/*/share/cfi_blacklist.txt', - ] + want = [ + 'bin/llvm-pdbutil' + exe_ext, + 'bin/llvm-symbolizer' + exe_ext, + 'bin/llvm-undname' + exe_ext, + # Copy built-in headers (lib/clang/3.x.y/include). + 'lib/clang/$V/include/*', + 'lib/clang/$V/share/asan_blacklist.txt', + 'lib/clang/$V/share/cfi_blacklist.txt', + ] if sys.platform == 'win32': - want.append('bin/clang-cl.exe') - want.append('bin/lld-link.exe') + want.extend([ + 'bin/clang-cl.exe', + 'bin/lld-link.exe', + ]) else: - want.append('bin/clang') + want.extend([ + 'bin/clang', + + # Include libclang_rt.builtins.a for Fuchsia targets. + 'lib/clang/$V/aarch64-fuchsia/lib/libclang_rt.builtins.a', + 'lib/clang/$V/x86_64-fuchsia/lib/libclang_rt.builtins.a', + ]) if sys.platform == 'darwin': want.extend([ - # AddressSanitizer runtime. - 'lib/clang/*/lib/darwin/libclang_rt.asan_iossim_dynamic.dylib', - 'lib/clang/*/lib/darwin/libclang_rt.asan_osx_dynamic.dylib', + # AddressSanitizer runtime. + 'lib/clang/$V/lib/darwin/libclang_rt.asan_iossim_dynamic.dylib', + 'lib/clang/$V/lib/darwin/libclang_rt.asan_osx_dynamic.dylib', - # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). - 'lib/clang/*/lib/darwin/libclang_rt.fuzzer_no_main_osx.a', + # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). + 'lib/clang/$V/lib/darwin/libclang_rt.fuzzer_no_main_osx.a', - # OS X and iOS builtin libraries (iossim is lipo'd into ios) for the - # _IsOSVersionAtLeast runtime function. - 'lib/clang/*/lib/darwin/libclang_rt.ios.a', - 'lib/clang/*/lib/darwin/libclang_rt.osx.a', + # OS X and iOS builtin libraries (iossim is lipo'd into ios) for the + # _IsOSVersionAtLeast runtime function. + 'lib/clang/$V/lib/darwin/libclang_rt.ios.a', + 'lib/clang/$V/lib/darwin/libclang_rt.osx.a', - # Profile runtime (used by profiler and code coverage). - 'lib/clang/*/lib/darwin/libclang_rt.profile_iossim.a', - 'lib/clang/*/lib/darwin/libclang_rt.profile_osx.a', + # Profile runtime (used by profiler and code coverage). + 'lib/clang/$V/lib/darwin/libclang_rt.profile_iossim.a', + 'lib/clang/$V/lib/darwin/libclang_rt.profile_osx.a', ]) elif sys.platform.startswith('linux'): - # Add llvm-ar and lld for LTO. - want.append('bin/llvm-ar') - want.append('bin/lld') want.extend([ - # AddressSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.asan-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.asan-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.asan-x86_64.a.syms', - - # AddressSanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.asan_cxx-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.asan_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.asan_cxx-x86_64.a.syms', - - # AddressSanitizer Android runtime. - 'lib/clang/*/lib/linux/libclang_rt.asan-aarch64-android.so', - 'lib/clang/*/lib/linux/libclang_rt.asan-arm-android.so', - 'lib/clang/*/lib/linux/libclang_rt.asan-i686-android.so', - - # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). - 'lib/clang/*/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a', - - # HWASAN Android runtime. - 'lib/clang/*/lib/linux/libclang_rt.hwasan-aarch64-android.so', - - # MemorySanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.msan-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.msan-x86_64.a.syms', - - # MemorySanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.msan_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.msan_cxx-x86_64.a.syms', - - # Profile runtime (used by profiler and code coverage). - 'lib/clang/*/lib/linux/libclang_rt.profile-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.profile-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.profile-aarch64-android.a', - 'lib/clang/*/lib/linux/libclang_rt.profile-arm-android.a', - - # ThreadSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.tsan-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.tsan-x86_64.a.syms', - - # ThreadSanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.tsan_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.tsan_cxx-x86_64.a.syms', - - # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-x86_64.a.syms', - - # UndefinedBehaviorSanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone_cxx-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a.syms', - - # UndefinedBehaviorSanitizer Android runtime, needed for CFI. - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-aarch64-android.so', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-arm-android.so', - - # Blacklist for MemorySanitizer (used on Linux only). - 'lib/clang/*/share/msan_blacklist.txt', + 'bin/lld', + + # Add llvm-ar for LTO. + 'bin/llvm-ar', + + # AddressSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.asan-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.asan-x86_64.a.syms', + + # AddressSanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.asan_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.asan_cxx-x86_64.a.syms', + + # AddressSanitizer Android runtime. + 'lib/clang/$V/lib/linux/libclang_rt.asan-aarch64-android.so', + 'lib/clang/$V/lib/linux/libclang_rt.asan-arm-android.so', + 'lib/clang/$V/lib/linux/libclang_rt.asan-i686-android.so', + + # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). + 'lib/clang/$V/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a', + + # HWASAN Android runtime. + 'lib/clang/$V/lib/linux/libclang_rt.hwasan-aarch64-android.so', + + # MemorySanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.msan-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.msan-x86_64.a.syms', + + # MemorySanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.msan_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.msan_cxx-x86_64.a.syms', + + # Profile runtime (used by profiler and code coverage). + 'lib/clang/$V/lib/linux/libclang_rt.profile-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.profile-aarch64-android.a', + 'lib/clang/$V/lib/linux/libclang_rt.profile-arm-android.a', + + # ThreadSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.tsan-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.tsan-x86_64.a.syms', + + # ThreadSanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.tsan_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.tsan_cxx-x86_64.a.syms', + + # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-x86_64.a.syms', + + # UndefinedBehaviorSanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a.syms', + + # UndefinedBehaviorSanitizer Android runtime, needed for CFI. + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-aarch64-android.so', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-arm-android.so', + + # Blacklist for MemorySanitizer (used on Linux only). + 'lib/clang/$V/share/msan_blacklist.txt', ]) elif sys.platform == 'win32': want.extend([ - # AddressSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/windows/clang_rt.asan-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan-x86_64.lib', - - # AddressSanitizer C++ runtime. - 'lib/clang/*/lib/windows/clang_rt.asan_cxx-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_cxx-x86_64.lib', - - # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). - 'lib/clang/*/lib/windows/clang_rt.fuzzer_no_main-x86_64.lib', - - # Thunk for AddressSanitizer needed for static build of a shared lib. - 'lib/clang/*/lib/windows/clang_rt.asan_dll_thunk-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_dll_thunk-x86_64.lib', - - # AddressSanitizer runtime for component build. - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-i386.dll', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-x86_64.dll', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-x86_64.lib', - - # Thunk for AddressSanitizer for component build of a shared lib. - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic_runtime_thunk-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic_runtime_thunk-x86_64.lib', - - # Profile runtime (used by profiler and code coverage). - 'lib/clang/*/lib/windows/clang_rt.profile-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.profile-x86_64.lib', - - # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone-x86_64.lib', - - # UndefinedBehaviorSanitizer C++ runtime. - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone_cxx-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone_cxx-x86_64.lib', + # AddressSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/windows/clang_rt.asan-x86_64.lib', + + # AddressSanitizer C++ runtime. + 'lib/clang/$V/lib/windows/clang_rt.asan_cxx-x86_64.lib', + + # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). + 'lib/clang/$V/lib/windows/clang_rt.fuzzer_no_main-x86_64.lib', + + # Thunk for AddressSanitizer needed for static build of a shared lib. + 'lib/clang/$V/lib/windows/clang_rt.asan_dll_thunk-x86_64.lib', + + # AddressSanitizer runtime for component build. + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-x86_64.dll', + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-x86_64.lib', + + # Thunk for AddressSanitizer for component build of a shared lib. + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic_runtime_thunk-x86_64.lib', + + # Profile runtime (used by profiler and code coverage). + 'lib/clang/$V/lib/windows/clang_rt.profile-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.profile-x86_64.lib', + + # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone-x86_64.lib', + + # UndefinedBehaviorSanitizer C++ runtime. + 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone_cxx-x86_64.lib', ]) - if sys.platform in ('linux2', 'darwin'): - # Include libclang_rt.builtins.a for Fuchsia targets. - want.extend(['lib/clang/*/aarch64-fuchsia/lib/libclang_rt.builtins.a', - 'lib/clang/*/x86_64-fuchsia/lib/libclang_rt.builtins.a', - ]) + # Check all non-glob wanted files exist on disk. + want = [w.replace('$V', RELEASE_VERSION) for w in want] + for w in want: + if '*' in w: continue + if os.path.exists(os.path.join(LLVM_RELEASE_DIR, w)): continue + print >>sys.stderr, 'wanted file "%s" but it did not exist' % w + return 1 + # TODO(thakis): Try walking over want and copying the files in there instead + # of walking the directory and doing fnmatch() against want. for root, dirs, files in os.walk(LLVM_RELEASE_DIR): # root: third_party/llvm-build/Release+Asserts/lib/..., rel_root: lib/... rel_root = root[len(LLVM_RELEASE_DIR)+1:] @@ -419,18 +402,20 @@ def main(): shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'), os.path.join(pdir, 'include', 'c++')) - # Copy buildlog over. - shutil.copy('buildlog.txt', pdir) - - # Create archive. - tar_entries = ['bin', 'lib', 'buildlog.txt'] + # Create main archive. + tar_entries = ['bin', 'lib' ] if sys.platform == 'darwin': tar_entries += ['include'] with tarfile.open(pdir + '.tgz', 'w:gz') as tar: for entry in tar_entries: tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress) + MaybeUpload(args.upload, pdir + '.tgz', platform) - MaybeUpload(args, pdir, platform) + # Upload build log next to it. + os.rename('buildlog.txt', pdir + '-buildlog.txt') + MaybeUpload(args.upload, pdir + '-buildlog.txt', platform, + extra_gsutil_args=['-z', 'txt']) + os.remove(pdir + '-buildlog.txt') # Zip up llvm-code-coverage for code coverage. code_coverage_dir = 'llvm-code-coverage-' + stamp @@ -442,7 +427,7 @@ def main(): with tarfile.open(code_coverage_dir + '.tgz', 'w:gz') as tar: tar.add(os.path.join(code_coverage_dir, 'bin'), arcname='bin', filter=PrintTarProgress) - MaybeUpload(args, code_coverage_dir, platform) + MaybeUpload(args.upload, code_coverage_dir + '.tgz', platform) # Zip up llvm-objdump and related tools for sanitizer coverage and Supersize. objdumpdir = 'llvmobjdump-' + stamp @@ -464,7 +449,7 @@ def main(): filter=PrintTarProgress) tar.add(llvmobjdump_stamp_file, arcname=llvmobjdump_stamp_file_base, filter=PrintTarProgress) - MaybeUpload(args, objdumpdir, platform) + MaybeUpload(args.upload, objdumpdir + '.tgz', platform) # On Mac, lld isn't part of the main zip. Upload it in a separate zip. if sys.platform == 'darwin': @@ -480,7 +465,7 @@ def main(): with tarfile.open(llddir + '.tgz', 'w:gz') as tar: tar.add(os.path.join(llddir, 'bin'), arcname='bin', filter=PrintTarProgress) - MaybeUpload(args, llddir, platform) + MaybeUpload(args.upload, llddir + '.tgz', platform) # dsymutil isn't part of the main zip either, and it gets periodically # deployed to CIPD (manually, not as part of clang rolls) for use in the @@ -493,7 +478,7 @@ def main(): with tarfile.open(dsymdir + '.tgz', 'w:gz') as tar: tar.add(os.path.join(dsymdir, 'bin'), arcname='bin', filter=PrintTarProgress) - MaybeUpload(args, dsymdir, platform) + MaybeUpload(args.upload, dsymdir + '.tgz', platform) # Zip up the translation_unit tool. translation_unit_dir = 'translation_unit-' + stamp @@ -505,7 +490,7 @@ def main(): with tarfile.open(translation_unit_dir + '.tgz', 'w:gz') as tar: tar.add(os.path.join(translation_unit_dir, 'bin'), arcname='bin', filter=PrintTarProgress) - MaybeUpload(args, translation_unit_dir, platform) + MaybeUpload(args.upload, translation_unit_dir + '.tgz', platform) if sys.platform == 'win32' and args.upload: UploadPDBToSymbolServer() |