# Copyright (c) 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Bisect repackage tool for Linux. This script repacakges chrome builds for manual bisect script. """ from functools import partial import json import logging from multiprocessing import Pool import optparse import os import re import sys import tempfile import threading import urllib import bisect_repackage_utils import re # This script uses cloud_storage module which contains gsutils wrappers. # cloud_storage module is a part of catapult repo, so please make sure # catapult is checked out before running this script. _PY_UTILS_PATH = os.path.abspath(os.path.join( os.path.dirname(__file__), '..', '..', 'third_party', 'catapult', 'common', 'py_utils')) if _PY_UTILS_PATH not in sys.path: sys.path.insert(1, _PY_UTILS_PATH) from py_utils import cloud_storage # Declares required files to run manual bisect script on chrome Linux # builds in perf. Binary files that should be stripped to reduce zip file # size are declared. The file list was gotten from the local chrome # executable path. (This can be retrieved by typing 'chrome://version' # in chrome and following the executable path. The list needs to be updated if # future chrome versions require additional files. CHROME_REQUIRED_FILES = { 'arm': ['apks/'], 'arm64': ['apks/'], 'linux': [ 'chrome', 'chrome_100_percent.pak', 'chrome_200_percent.pak', 'chromedriver', 'default_apps/', 'icudtl.dat', 'libclearkeycdm.so', 'libwidevinecdm.so', 'locales/', 'nacl_helper', 'nacl_helper_bootstrap', 'nacl_helper_nonsfi', 'nacl_irt_x86_64.nexe', 'natives_blob.bin', 'pnacl/', 'product_logo_48.png', 'resources/', 'resources.pak', 'v8_context_snapshot.bin', 'xdg-mime', 'xdg-settings' ], 'win64': [ 'chrome.dll', 'chrome.exe', 'chrome_100_percent.pak', 'chrome_200_percent.pak', 'chrome_child.dll', 'chrome_elf.dll', 'chrome_watcher.dll', 'chromedriver.exe', 'default_apps', 'd3dcompiler_47.dll', 'icudtl.dat', 'libEGL.dll', 'libGLESv2.dll', 'locales', 'nacl_irt_x86_64.nexe', 'natives_blob.bin', 'PepperFlash', 'resources.pak', 'SecondaryTile.png', 'v8_context_snapshot.bin' ], 'mac': [ 'chromedriver', 'Google Chrome.app' ] } CHROME_WHITELIST_FILES = { 'win64': '^\d+\.\d+\.\d+\.\d+\.manifest$', } # No stripping symbols from android, windows or mac archives CHROME_STRIP_LIST = { 'linux': [ 'chrome', 'nacl_helper' ] } # API to convert Githash to Commit position number. CHROMIUM_GITHASH_TO_SVN_URL = ( 'https://cr-rev.appspot.com/_ah/api/crrev/v1/commit/%s') CHROMIUM_CP_TO_GITHASH = ( 'https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/%s') REVISION_MAP_FILE = 'revision_map.json' BUILDER_NAME = { 'arm': 'Android Builder', 'arm64': 'Android arm64 Builder', 'linux': 'Linux Builder', 'mac': 'Mac Builder', 'win32': 'Win Builder', 'win64': 'Win x64 Builder' } ARCHIVE_PREFIX = { 'arm': 'full-build-linux', 'arm64': 'full-build-linux', 'linux': 'full-build-linux', 'mac': 'full-build-mac', 'win32': 'full-build-win32', 'win64': 'full-build-win32' } CHROME_TEST_BUCKET_SUBFOLDER = 'official-by-commit' class ChromeExecutionError(Exception): """Raised when Chrome execution fails.""" pass class GitConversionError(Exception): """Raised when Chrome execution fails.""" pass class PathContext(object): """Stores information to repackage from a bucket to another. A PathContext is used to carry the information used to construct URLs and paths when dealing with the storage server and archives. """ def __init__(self, source_bucket, repackage_bucket, archive, revision_file=REVISION_MAP_FILE): super(PathContext, self).__init__() self.archive = archive self.builder_name = BUILDER_NAME[archive] self.original_gs_bucket = source_bucket self.original_remote_path = BUILDER_NAME[archive] self.repackage_gs_bucket = repackage_bucket self.repackage_remote_path = '%s/%s' % (CHROME_TEST_BUCKET_SUBFOLDER, BUILDER_NAME[archive]) self.file_prefix = ARCHIVE_PREFIX[archive] self.revision_file = os.path.join(os.getcwd(), revision_file) def GetExtractedDir(self): # Perf builders archives the binaries in out/Release directory. if self.archive in ['arm', 'arm64']: return os.path.join('out', 'Release') return self.file_prefix def get_cp_from_hash(git_hash): """Converts a git hash to commit position number.""" json_url = CHROMIUM_GITHASH_TO_SVN_URL % git_hash response = urllib.urlopen(json_url) if response.getcode() == 200: try: data = json.loads(response.read()) except Exception,e: logging.warning('JSON URL: %s, Error Message: %s' % json_url, e) raise GitConversionError else: logging.warning('JSON URL: %s, Error Message: %s' % json_url, e) raise GitConversionError if 'number' in data: return data['number'] logging.warning('JSON URL: %s, Error Message: %s' % json_url, e) raise GitConversionError def create_cp_from_hash_map(hash_list): """Returns dict used for conversion of hash list. Creates a dictionary that maps from Commit position number to corresponding GitHash. """ hash_map = {} for git_hash in hash_list: try: cp_num = get_cp_from_hash(git_hash) hash_map[cp_num] = git_hash except GitConversionError: pass return hash_map def get_list_of_suffix(bucket_address, prefix, filter_function): """Gets the list of suffixes in files in a google storage bucket. Example: a google storage bucket containing one file 'full-build-linux_20983' will return ['20983'] if prefix is provided as 'full-build-linux'. Google Storage bucket containing multiple files will return multiple suffixes. Args: bucket_address(String): Bucket URL to examine files from. prefix(String): The prefix used in creating build file names filter_function: A function that returns true if the extracted suffix is in correct format and false otherwise. It allows only proper suffix to be extracted and returned. Returns: (List) list of proper suffixes in the bucket. """ file_list = cloud_storage.List(bucket_address) suffix_list = [] extract_suffix = '.*?%s_(.*?)\.zip' %(prefix) for file in file_list: match = re.match(extract_suffix, file) if match and filter_function(match.groups()[0]): suffix_list.append(match.groups()[0]) return suffix_list def download_build(cp_num, revision_map, zip_file_name, context): """Download a single build corresponding to the cp_num and context.""" remote_file_path = '%s/%s_%s.zip' % (context.original_remote_path, context.file_prefix, revision_map[cp_num]) try: cloud_storage.Get(context.original_gs_bucket, remote_file_path, zip_file_name) except Exception, e: logging.warning('Failed to download: %s, error: %s', zip_file_name, e) return False return True def upload_build(zip_file, context): """Uploads a single build in zip_file to the repackage_gs_url in context.""" cloud_storage.Insert( context.repackage_gs_bucket, context.repackage_remote_path, zip_file) def download_revision_map(context): """Downloads the revision map in original_gs_url in context.""" download_file = '%s/%s' % (context.repackage_remote_path, REVISION_MAP_FILE) cloud_storage.Get(context.repackage_gs_bucket, download_file, context.revision_file) def get_revision_map(context): """Downloads and returns the revision map in repackage_gs_url in context.""" bisect_repackage_utils.RemoveFile(context.revision_file) download_revision_map(context) with open(context.revision_file, 'r') as revision_file: revision_map = json.load(revision_file) bisect_repackage_utils.RemoveFile(context.revision_file) return revision_map def upload_revision_map(revision_map, context): """Upload the given revision_map to the repackage_gs_url in context.""" with open(context.revision_file, 'w') as revision_file: json.dump(revision_map, revision_file) cloud_storage.Insert(context.repackage_gs_bucket, context.repackage_remote_path, context.revision_file) bisect_repackage_utils.RemoveFile(context.revision_file) def create_upload_revision_map(context): """Creates and uploads a dictionary that maps from GitHash to CP number.""" gs_base_url = '%s/%s' % (context.original_gs_bucket, context.original_remote_path) hash_list = get_list_of_suffix(gs_base_url, context.file_prefix, bisect_repackage_utils.IsGitCommitHash) cp_num_to_hash_map = create_cp_from_hash_map(hash_list) upload_revision_map(cp_num_to_hash_map, context) def update_upload_revision_map(context): """Updates and uploads a dictionary that maps from GitHash to CP number.""" gs_base_url = '%s/%s' % (context.original_gs_bucket, context.original_remote_path) revision_map = get_revision_map(context) hash_list = get_list_of_suffix(gs_base_url, context.file_prefix, bisect_repackage_utils.IsGitCommitHash) hash_list = list(set(hash_list)-set(revision_map.values())) cp_num_to_hash_map = create_cp_from_hash_map(hash_list) merged_dict = dict(cp_num_to_hash_map.items() + revision_map.items()) upload_revision_map(merged_dict, context) def make_lightweight_archive(file_archive, archive_name, files_to_archive, context, staging_dir, ignore_sub_folder): """Repackages and strips the archive. Repacakges and strips according to CHROME_REQUIRED_FILES and CHROME_STRIP_LIST. """ strip_list = CHROME_STRIP_LIST.get(context.archive) tmp_archive = os.path.join(staging_dir, 'tmp_%s' % archive_name) (zip_dir, zip_file) = bisect_repackage_utils.MakeZip( tmp_archive, archive_name, files_to_archive, file_archive, dir_in_zip=context.GetExtractedDir(), raise_error=False, strip_files=strip_list, ignore_sub_folder=ignore_sub_folder) return (zip_dir, zip_file, tmp_archive) def remove_created_files_and_path(files, paths): """Removes all the files and paths passed in.""" for file in files: bisect_repackage_utils.RemoveFile(file) for path in paths: bisect_repackage_utils.RemovePath(path) def verify_chrome_run(zip_dir): """This function executes chrome executable in zip_dir. Currently, it is only supported for Linux Chrome builds. Raises error if the execution fails for any reason. """ try: command = [os.path.join(zip_dir, 'chrome')] code = bisect_repackage_utils.RunCommand(command) if code != 0: raise ChromeExecutionError('An error occurred when executing Chrome') except ChromeExecutionError,e: print str(e) def get_whitelist_files(extracted_folder, archive): """Gets all the files & directories matching whitelisted regex.""" whitelist_files = [] all_files = os.listdir(extracted_folder) for file in all_files: if re.match(CHROME_WHITELIST_FILES.get(archive), file): whitelist_files.append(file) return whitelist_files def repackage_single_revision(revision_map, verify_run, staging_dir, context, cp_num): """Repackages a single Chrome build for manual bisect.""" archive_name = '%s_%s' %(context.file_prefix, cp_num) file_archive = os.path.join(staging_dir, archive_name) zip_file_name = '%s.zip' % (file_archive) if not download_build(cp_num, revision_map, zip_file_name, context): return extract_dir = os.path.join(staging_dir, archive_name) is_android = context.archive in ['arm', 'arm64'] files_to_include = CHROME_REQUIRED_FILES.get(context.archive) dir_path_in_zip = context.GetExtractedDir() extract_file_list = [] # Only extract required files and directories. # And when there is no pattern checking for files. if not CHROME_WHITELIST_FILES.get(context.archive): for f in files_to_include: if f.endswith('/'): f += '*' extract_file_list.append(os.path.join(dir_path_in_zip, f)) bisect_repackage_utils.ExtractZip( zip_file_name, extract_dir, extract_file_list) extracted_folder = os.path.join(extract_dir, dir_path_in_zip) if CHROME_WHITELIST_FILES.get(context.archive): whitelist_files = get_whitelist_files(extracted_folder, context.archive) files_to_include += whitelist_files (zip_dir, zip_file, tmp_archive) = make_lightweight_archive(extracted_folder, archive_name, files_to_include, context, staging_dir, is_android) if verify_run: verify_chrome_run(zip_dir) upload_build(zip_file, context) with open('upload_revs.json', 'r+') as rfile: update_map = json.load(rfile) update_map[str(cp_num)] = 'Done' rfile.seek(0) json.dump(update_map, rfile) rfile.truncate() # Removed temporary files created during repackaging process. remove_created_files_and_path([zip_file_name], [zip_dir, extract_dir, tmp_archive]) def repackage_revisions(revisions, revision_map, verify_run, staging_dir, context, quit_event=None, progress_event=None): """Repackages all Chrome builds listed in revisions. This function calls 'repackage_single_revision' with multithreading pool. """ p = Pool(3) func = partial(repackage_single_revision, revision_map, verify_run, staging_dir, context) p.imap(func, revisions) p.close() p.join() def get_uploaded_builds(context): """Returns already uploaded revisions in original bucket.""" gs_base_url = '%s/%s' % (context.repackage_gs_bucket, context.repackage_remote_path) return get_list_of_suffix(gs_base_url, context.file_prefix, bisect_repackage_utils.IsCommitPosition) def get_revisions_to_package(revision_map, context): """Returns revisions that need to be repackaged. It subtracts revisions that are already packaged from all revisions that need to be packaged. The revisions will be sorted in descending order. """ already_packaged = get_uploaded_builds(context) not_already_packaged = list(set(revision_map.keys())-set(already_packaged)) revisions_to_package = sorted(not_already_packaged, reverse=True) return revisions_to_package def get_hash_from_cp(cp_num): """Converts a commit position number to git hash.""" json_url = CHROMIUM_CP_TO_GITHASH % cp_num response = urllib.urlopen(json_url) if response.getcode() == 200: try: data = json.loads(response.read()) if 'git_sha' in data: return data['git_sha'] except Exception, e: logging.warning('Failed to fetch git_hash: %s, error: %s' % json_url, e) else: logging.warning('Failed to fetch git_hash: %s, CP: %s' % json_url, cp_num) return None def get_revision_map_for_range(start_rev, end_rev): revision_map = {} for cp_num in range(start_rev, end_rev + 1 ): git_hash = get_hash_from_cp(cp_num) if git_hash: revision_map[cp_num] = git_hash return revision_map def get_overwrite_revisions(revision_map): return sorted(revision_map.keys(), reverse=True) class RepackageJob(object): def __init__(self, name, revisions_to_package, revision_map, verify_run, staging_dir, context): super(RepackageJob, self).__init__() self.name = name self.revisions_to_package = revisions_to_package self.revision_map = revision_map self.verify_run = verify_run self.staging_dir = staging_dir self.context = context self.quit_event = threading.Event() self.progress_event = threading.Event() self.thread = None def Start(self): """Starts the download.""" fetchargs = (self.revisions_to_package, self.revision_map, self.verify_run, self.staging_dir, self.context, self.quit_event, self.progress_event) self.thread = threading.Thread(target=repackage_revisions, name=self.name, args=fetchargs) self.thread.start() def Stop(self): """Stops the download which must have been started previously.""" assert self.thread, 'DownloadJob must be started before Stop is called.' self.quit_event.set() self.thread.join() def WaitFor(self): """Prints a message and waits for the download to complete.""" assert self.thread, 'DownloadJob must be started before WaitFor is called.' self.progress_event.set() # Display progress of download. def Stop(self): assert self.thread, 'DownloadJob must be started before Stop is called.' self.quit_event.set() self.thread.join() def main(argv): option_parser = optparse.OptionParser() choices = ['mac', 'win32', 'win64', 'linux', 'arm', 'arm64'] option_parser.add_option('-a', '--archive', choices=choices, help='Builders to repacakge from [%s].' % '|'.join(choices)) # Verifies that the chrome executable runs option_parser.add_option('-v', '--verify', action='store_true', help='Verifies that the Chrome executes normally' 'without errors') # This option will update the revision map. option_parser.add_option('-u', '--update', action='store_true', help='Updates the list of revisions to repackage') # This option will creates the revision map. option_parser.add_option('-c', '--create', action='store_true', help='Creates the list of revisions to repackage') # Original bucket that contains perf builds option_parser.add_option('-o', '--original', type='str', help='Google storage bucket name containing original' 'Chrome builds') # Bucket that should archive lightweight perf builds option_parser.add_option('-r', '--repackage', type='str', help='Google storage bucket name ' 'to re-archive Chrome builds') # Overwrites build archives for a given range. option_parser.add_option('-w', '--overwrite', action='store_true', dest='overwrite', help='Overwrite build archives') # Start revision for build overwrite. option_parser.add_option('-s', '--start_rev', type='str', dest='start_rev', help='Start revision for overwrite') # Start revision for build overwrite. option_parser.add_option('-e', '--end_rev', type='str', dest='end_rev', help='end revision for overwrite') verify_run = False (opts, args) = option_parser.parse_args() if opts.archive is None: print 'Error: missing required parameter: --archive' option_parser.print_help() return 1 if not opts.original or not opts.repackage: raise ValueError('Need to specify original gs bucket url and' 'repackage gs bucket url') context = PathContext(opts.original, opts.repackage, opts.archive) if opts.create: create_upload_revision_map(context) if opts.update: update_upload_revision_map(context) if opts.verify: verify_run = True if opts.overwrite: if not opts.start_rev or not opts.end_rev: raise ValueError('Need to specify overwrite range start (-s) and end (-e)' ' revision.') revision_map = get_revision_map_for_range( int(opts.start_rev), int(opts.end_rev)) backward_rev = get_overwrite_revisions(revision_map) with open('upload_revs.json', 'w') as revision_file: json.dump(revision_map, revision_file) else: revision_map = get_revision_map(context) backward_rev = get_revisions_to_package(revision_map, context) base_dir = os.path.join('.', context.archive) # Clears any uncleared staging directories and create one bisect_repackage_utils.RemovePath(base_dir) bisect_repackage_utils.MaybeMakeDirectory(base_dir) staging_dir = os.path.abspath(tempfile.mkdtemp(prefix='staging', dir=base_dir)) repackage = RepackageJob('backward_fetch', backward_rev, revision_map, verify_run, staging_dir, context) # Multi-threading is not currently being used. But it can be used in # cases when the repackaging needs to be quicker. try: repackage.Start() repackage.WaitFor() except (KeyboardInterrupt, SystemExit): print 'Cleaning up...' bisect_repackage_utils.RemovePath(staging_dir) print 'Cleaning up...' bisect_repackage_utils.RemovePath(staging_dir) if '__main__' == __name__: sys.exit(main(sys.argv))