diff options
Diffstat (limited to 'deps/v8/tools/push-to-trunk')
-rwxr-xr-x | deps/v8/tools/push-to-trunk/auto_roll.py | 23 | ||||
-rwxr-xr-x | deps/v8/tools/push-to-trunk/auto_tag.py | 200 | ||||
-rwxr-xr-x | deps/v8/tools/push-to-trunk/bump_up_version.py | 241 | ||||
-rwxr-xr-x | deps/v8/tools/push-to-trunk/chromium_roll.py | 6 | ||||
-rw-r--r-- | deps/v8/tools/push-to-trunk/common_includes.py | 55 | ||||
-rw-r--r-- | deps/v8/tools/push-to-trunk/git_recipes.py | 13 | ||||
-rwxr-xr-x | deps/v8/tools/push-to-trunk/push_to_trunk.py | 46 | ||||
-rwxr-xr-x | deps/v8/tools/push-to-trunk/releases.py | 68 | ||||
-rw-r--r-- | deps/v8/tools/push-to-trunk/test_scripts.py | 218 |
9 files changed, 824 insertions, 46 deletions
diff --git a/deps/v8/tools/push-to-trunk/auto_roll.py b/deps/v8/tools/push-to-trunk/auto_roll.py index 607ca0897..6e6c7fe2a 100755 --- a/deps/v8/tools/push-to-trunk/auto_roll.py +++ b/deps/v8/tools/push-to-trunk/auto_roll.py @@ -12,8 +12,11 @@ import urllib from common_includes import * import chromium_roll +CLUSTERFUZZ_API_KEY_FILE = "CLUSTERFUZZ_API_KEY_FILE" + CONFIG = { PERSISTFILE_BASENAME: "/tmp/v8-auto-roll-tempfile", + CLUSTERFUZZ_API_KEY_FILE: ".cf_api_key", } CR_DEPS_URL = 'http://src.chromium.org/svn/trunk/src/DEPS' @@ -65,6 +68,24 @@ class DetectLastRoll(Step): return True +class CheckClusterFuzz(Step): + MESSAGE = "Check ClusterFuzz api for new problems." + + def RunStep(self): + if not os.path.exists(self.Config(CLUSTERFUZZ_API_KEY_FILE)): + print "Skipping ClusterFuzz check. No api key file found." + return False + api_key = FileToText(self.Config(CLUSTERFUZZ_API_KEY_FILE)) + # Check for open, reproducible issues that have no associated bug. + result = self._side_effect_handler.ReadClusterFuzzAPI( + api_key, job_type="linux_asan_d8_dbg", reproducible="True", + open="True", bug_information="", + revision_greater_or_equal=str(self["last_push"])) + if result: + print "Stop due to pending ClusterFuzz issues." + return True + + class RollChromium(Step): MESSAGE = "Roll V8 into Chromium." @@ -75,6 +96,7 @@ class RollChromium(Step): "--reviewer", self._options.reviewer, "--chromium", self._options.chromium, "--force", + "--use-commit-queue", ] if self._options.sheriff: args.extend([ @@ -108,6 +130,7 @@ class AutoRoll(ScriptsBase): CheckActiveRoll, DetectLastPush, DetectLastRoll, + CheckClusterFuzz, RollChromium, ] diff --git a/deps/v8/tools/push-to-trunk/auto_tag.py b/deps/v8/tools/push-to-trunk/auto_tag.py new file mode 100755 index 000000000..6beaaff8a --- /dev/null +++ b/deps/v8/tools/push-to-trunk/auto_tag.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import sys + +from common_includes import * + +CONFIG = { + BRANCHNAME: "auto-tag-v8", + PERSISTFILE_BASENAME: "/tmp/v8-auto-tag-tempfile", + DOT_GIT_LOCATION: ".git", + VERSION_FILE: "src/version.cc", +} + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + self.CommonPrepare() + self.PrepareBranch() + self.GitCheckout("master") + self.GitSVNRebase() + + +class GetTags(Step): + MESSAGE = "Get all V8 tags." + + def RunStep(self): + self.GitCreateBranch(self._config[BRANCHNAME]) + + # Get remote tags. + tags = filter(lambda s: re.match(r"^svn/tags/[\d+\.]+$", s), + self.GitRemotes()) + + # Remove 'svn/tags/' prefix. + self["tags"] = map(lambda s: s[9:], tags) + + +class GetOldestUntaggedVersion(Step): + MESSAGE = "Check if there's a version on bleeding edge without a tag." + + def RunStep(self): + tags = set(self["tags"]) + self["candidate"] = None + self["candidate_version"] = None + self["next"] = None + self["next_version"] = None + + # Iterate backwards through all automatic version updates. + for git_hash in self.GitLog( + format="%H", grep="\\[Auto\\-roll\\] Bump up version to").splitlines(): + + # Get the version. + if not self.GitCheckoutFileSafe(self._config[VERSION_FILE], git_hash): + continue + + self.ReadAndPersistVersion() + version = self.ArrayToVersion("") + + # Strip off trailing patch level (tags don't include tag level 0). + if version.endswith(".0"): + version = version[:-2] + + # Clean up checked-out version file. + self.GitCheckoutFileSafe(self._config[VERSION_FILE], "HEAD") + + if version in tags: + if self["candidate"]: + # Revision "git_hash" is tagged already and "candidate" was the next + # newer revision without a tag. + break + else: + print("Stop as %s is the latest version and it has been tagged." % + version) + self.CommonCleanup() + return True + else: + # This is the second oldest version without a tag. + self["next"] = self["candidate"] + self["next_version"] = self["candidate_version"] + + # This is the oldest version without a tag. + self["candidate"] = git_hash + self["candidate_version"] = version + + if not self["candidate"] or not self["candidate_version"]: + print "Nothing found to tag." + self.CommonCleanup() + return True + + print("Candidate for tagging is %s with version %s" % + (self["candidate"], self["candidate_version"])) + + +class GetLKGRs(Step): + MESSAGE = "Get the last lkgrs." + + def RunStep(self): + revision_url = "https://v8-status.appspot.com/revisions?format=json" + status_json = self.ReadURL(revision_url, wait_plan=[5, 20]) + self["lkgrs"] = [entry["revision"] + for entry in json.loads(status_json) if entry["status"]] + + +class CalculateTagRevision(Step): + MESSAGE = "Calculate the revision to tag." + + def LastLKGR(self, min_rev, max_rev): + """Finds the newest lkgr between min_rev (inclusive) and max_rev + (exclusive). + """ + for lkgr in self["lkgrs"]: + # LKGRs are reverse sorted. + if int(min_rev) <= int(lkgr) and int(lkgr) < int(max_rev): + return lkgr + return None + + def RunStep(self): + # Get the lkgr after the tag candidate and before the next tag candidate. + candidate_svn = self.GitSVNFindSVNRev(self["candidate"]) + if self["next"]: + next_svn = self.GitSVNFindSVNRev(self["next"]) + else: + # Don't include the version change commit itself if there is no upper + # limit yet. + candidate_svn = str(int(candidate_svn) + 1) + next_svn = sys.maxint + lkgr_svn = self.LastLKGR(candidate_svn, next_svn) + + if not lkgr_svn: + print "There is no lkgr since the candidate version yet." + self.CommonCleanup() + return True + + # Let's check if the lkgr is at least three hours old. + self["lkgr"] = self.GitSVNFindGitHash(lkgr_svn) + if not self["lkgr"]: + print "Couldn't find git hash for lkgr %s" % lkgr_svn + self.CommonCleanup() + return True + + lkgr_utc_time = int(self.GitLog(n=1, format="%at", git_hash=self["lkgr"])) + current_utc_time = self._side_effect_handler.GetUTCStamp() + + if current_utc_time < lkgr_utc_time + 10800: + print "Candidate lkgr %s is too recent for tagging." % lkgr_svn + self.CommonCleanup() + return True + + print "Tagging revision %s with %s" % (lkgr_svn, self["candidate_version"]) + + +class MakeTag(Step): + MESSAGE = "Tag the version." + + def RunStep(self): + if not self._options.dry_run: + self.GitReset(self["lkgr"]) + self.GitSVNTag(self["candidate_version"]) + + +class CleanUp(Step): + MESSAGE = "Clean up." + + def RunStep(self): + self.CommonCleanup() + + +class AutoTag(ScriptsBase): + def _PrepareOptions(self, parser): + parser.add_argument("--dry_run", help="Don't tag the new version.", + default=False, action="store_true") + + def _ProcessOptions(self, options): # pragma: no cover + if not options.dry_run and not options.author: + print "Specify your chromium.org email with -a" + return False + options.wait_for_lgtm = False + options.force_readline_defaults = True + options.force_upload = True + return True + + def _Steps(self): + return [ + Preparation, + GetTags, + GetOldestUntaggedVersion, + GetLKGRs, + CalculateTagRevision, + MakeTag, + CleanUp, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(AutoTag(CONFIG).Run()) diff --git a/deps/v8/tools/push-to-trunk/bump_up_version.py b/deps/v8/tools/push-to-trunk/bump_up_version.py new file mode 100755 index 000000000..af5f73a60 --- /dev/null +++ b/deps/v8/tools/push-to-trunk/bump_up_version.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Script for auto-increasing the version on bleeding_edge. + +The script can be run regularly by a cron job. It will increase the build +level of the version on bleeding_edge if: +- the lkgr version is smaller than the version of the latest revision, +- the lkgr version is not a version change itself, +- the tree is not closed for maintenance. + +The new version will be the maximum of the bleeding_edge and trunk versions +1. +E.g. latest bleeding_edge version: 3.22.11.0 and latest trunk 3.23.0.0 gives +the new version 3.23.1.0. + +This script requires a depot tools git checkout. I.e. 'fetch v8'. +""" + +import argparse +import os +import sys + +from common_includes import * + +CONFIG = { + PERSISTFILE_BASENAME: "/tmp/v8-bump-up-version-tempfile", + VERSION_FILE: "src/version.cc", +} + +VERSION_BRANCH = "auto-bump-up-version" + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + # Check for a clean workdir. + if not self.GitIsWorkdirClean(): # pragma: no cover + # This is in case a developer runs this script on a dirty tree. + self.GitStash() + + # TODO(machenbach): This should be called master after the git switch. + self.GitCheckout("bleeding_edge") + + self.GitPull() + + # Ensure a clean version branch. + self.DeleteBranch(VERSION_BRANCH) + + +class GetCurrentBleedingEdgeVersion(Step): + MESSAGE = "Get latest bleeding edge version." + + def RunStep(self): + # TODO(machenbach): This should be called master after the git switch. + self.GitCheckout("bleeding_edge") + + # Store latest version and revision. + self.ReadAndPersistVersion() + self["latest_version"] = self.ArrayToVersion("") + self["latest"] = self.GitLog(n=1, format="%H") + print "Bleeding edge version: %s" % self["latest_version"] + + +# This step is pure paranoia. It forbids the script to continue if the last +# commit changed version.cc. Just in case the other bailout has a bug, this +# prevents the script from continuously commiting version changes. +class LastChangeBailout(Step): + MESSAGE = "Stop script if the last change modified the version." + + def RunStep(self): + if self._config[VERSION_FILE] in self.GitChangedFiles(self["latest"]): + print "Stop due to recent version change." + return True + + +# TODO(machenbach): Implement this for git. +class FetchLKGR(Step): + MESSAGE = "Fetching V8 LKGR." + + def RunStep(self): + lkgr_url = "https://v8-status.appspot.com/lkgr" + self["lkgr_svn"] = self.ReadURL(lkgr_url, wait_plan=[5]) + + +# TODO(machenbach): Implement this for git. With a git lkgr we could simply +# checkout that revision. With svn, we have to search backwards until that +# revision is found. +class GetLKGRVersion(Step): + MESSAGE = "Get bleeding edge lkgr version." + + def RunStep(self): + self.GitCheckout("bleeding_edge") + # If the commit was made from svn, there is a mapping entry in the commit + # message. + self["lkgr"] = self.GitLog( + grep="^git-svn-id: [^@]*@%s [A-Za-z0-9-]*$" % self["lkgr_svn"], + format="%H") + + # FIXME(machenbach): http://crbug.com/391712 can lead to svn lkgrs on the + # trunk branch (rarely). + if not self["lkgr"]: # pragma: no cover + self.Die("No git hash found for svn lkgr.") + + self.GitCreateBranch(VERSION_BRANCH, self["lkgr"]) + self.ReadAndPersistVersion("lkgr_") + self["lkgr_version"] = self.ArrayToVersion("lkgr_") + print "LKGR version: %s" % self["lkgr_version"] + + # Ensure a clean version branch. + self.GitCheckout("bleeding_edge") + self.DeleteBranch(VERSION_BRANCH) + + +class LKGRVersionUpToDateBailout(Step): + MESSAGE = "Stop script if the lkgr has a renewed version." + + def RunStep(self): + # If a version-change commit becomes the lkgr, don't bump up the version + # again. + if self._config[VERSION_FILE] in self.GitChangedFiles(self["lkgr"]): + print "Stop because the lkgr is a version change itself." + return True + + # Don't bump up the version if it got updated already after the lkgr. + if SortingKey(self["lkgr_version"]) < SortingKey(self["latest_version"]): + print("Stop because the latest version already changed since the lkgr " + "version.") + return True + + +class GetTrunkVersion(Step): + MESSAGE = "Get latest trunk version." + + def RunStep(self): + # TODO(machenbach): This should be called trunk after the git switch. + self.GitCheckout("master") + self.GitPull() + self.ReadAndPersistVersion("trunk_") + self["trunk_version"] = self.ArrayToVersion("trunk_") + print "Trunk version: %s" % self["trunk_version"] + + +class CalculateVersion(Step): + MESSAGE = "Calculate the new version." + + def RunStep(self): + if self["lkgr_build"] == "9999": # pragma: no cover + # If version control on bleeding edge was switched off, just use the last + # trunk version. + self["lkgr_version"] = self["trunk_version"] + + # The new version needs to be greater than the max on bleeding edge and + # trunk. + max_version = max(self["trunk_version"], + self["lkgr_version"], + key=SortingKey) + + # Strip off possible leading zeros. + self["new_major"], self["new_minor"], self["new_build"], _ = ( + map(str, map(int, max_version.split(".")))) + + self["new_build"] = str(int(self["new_build"]) + 1) + self["new_patch"] = "0" + + self["new_version"] = ("%s.%s.%s.0" % + (self["new_major"], self["new_minor"], self["new_build"])) + print "New version is %s" % self["new_version"] + + if self._options.dry_run: # pragma: no cover + print "Dry run, skipping version change." + return True + + +class CheckTreeStatus(Step): + MESSAGE = "Checking v8 tree status message." + + def RunStep(self): + status_url = "https://v8-status.appspot.com/current?format=json" + status_json = self.ReadURL(status_url, wait_plan=[5, 20, 300, 300]) + message = json.loads(status_json)["message"] + if re.search(r"maintenance|no commits", message, flags=re.I): + print "Skip version change by tree status: \"%s\"" % message + return True + + +class ChangeVersion(Step): + MESSAGE = "Bump up the version." + + def RunStep(self): + self.GitCreateBranch(VERSION_BRANCH, "bleeding_edge") + + self.SetVersion(self.Config(VERSION_FILE), "new_") + + try: + self.GitCommit("[Auto-roll] Bump up version to %s\n\nTBR=%s" % + (self["new_version"], self._options.author)) + self.GitUpload(author=self._options.author, + force=self._options.force_upload, + bypass_hooks=True) + self.GitDCommit() + print "Successfully changed the version." + finally: + # Clean up. + self.GitCheckout("bleeding_edge") + self.DeleteBranch(VERSION_BRANCH) + + +class BumpUpVersion(ScriptsBase): + def _PrepareOptions(self, parser): + parser.add_argument("--dry_run", help="Don't commit the new version.", + default=False, action="store_true") + + def _ProcessOptions(self, options): # pragma: no cover + if not options.dry_run and not options.author: + print "Specify your chromium.org email with -a" + return False + options.wait_for_lgtm = False + options.force_readline_defaults = True + options.force_upload = True + return True + + def _Steps(self): + return [ + Preparation, + GetCurrentBleedingEdgeVersion, + LastChangeBailout, + FetchLKGR, + GetLKGRVersion, + LKGRVersionUpToDateBailout, + GetTrunkVersion, + CalculateVersion, + CheckTreeStatus, + ChangeVersion, + ] + +if __name__ == "__main__": # pragma: no cover + sys.exit(BumpUpVersion(CONFIG).Run()) diff --git a/deps/v8/tools/push-to-trunk/chromium_roll.py b/deps/v8/tools/push-to-trunk/chromium_roll.py index 35ab24b05..0138ff8e7 100755 --- a/deps/v8/tools/push-to-trunk/chromium_roll.py +++ b/deps/v8/tools/push-to-trunk/chromium_roll.py @@ -105,7 +105,8 @@ class UploadCL(Step): % self["sheriff"]) self.GitCommit("%s%s\n\nTBR=%s" % (commit_title, sheriff, rev)) self.GitUpload(author=self._options.author, - force=self._options.force_upload) + force=self._options.force_upload, + cq=self._options.use_commit_queue) print "CL uploaded." @@ -143,6 +144,9 @@ class ChromiumRoll(ScriptsBase): "directory to automate the V8 roll.")) parser.add_argument("-l", "--last-push", help="The git commit ID of the last push to trunk.") + parser.add_argument("--use-commit-queue", + help="Check the CQ bit on upload.", + default=False, action="store_true") def _ProcessOptions(self, options): # pragma: no cover if not options.manual and not options.reviewer: diff --git a/deps/v8/tools/push-to-trunk/common_includes.py b/deps/v8/tools/push-to-trunk/common_includes.py index 482509f7d..0e57a25bb 100644 --- a/deps/v8/tools/push-to-trunk/common_includes.py +++ b/deps/v8/tools/push-to-trunk/common_includes.py @@ -28,6 +28,7 @@ import argparse import datetime +import httplib import imp import json import os @@ -36,6 +37,7 @@ import subprocess import sys import textwrap import time +import urllib import urllib2 from git_recipes import GitRecipesMixin @@ -169,6 +171,16 @@ def MakeChangeLogBugReference(body): return "" +def SortingKey(version): + """Key for sorting version number strings: '3.11' > '3.2.1.1'""" + version_keys = map(int, version.split(".")) + # Fill up to full version numbers to normalize comparison. + while len(version_keys) < 4: # pragma: no cover + version_keys.append(0) + # Fill digits. + return ".".join(map("{0:04d}".format, version_keys)) + + # Some commands don't like the pipe, e.g. calling vi from within the script or # from subscripts like git cl upload. def Command(cmd, args="", prefix="", pipe=True): @@ -207,12 +219,34 @@ class SideEffectHandler(object): # pragma: no cover finally: url_fh.close() + def ReadClusterFuzzAPI(self, api_key, **params): + params["api_key"] = api_key.strip() + params = urllib.urlencode(params) + + headers = {"Content-type": "application/x-www-form-urlencoded"} + + conn = httplib.HTTPSConnection("backend-dot-cluster-fuzz.appspot.com") + conn.request("POST", "/_api/", params, headers) + + response = conn.getresponse() + data = response.read() + + try: + return json.loads(data) + except: + print data + print "ERROR: Could not read response. Is your key valid?" + raise + def Sleep(self, seconds): time.sleep(seconds) def GetDate(self): return datetime.date.today().strftime("%Y-%m-%d") + def GetUTCStamp(self): + return time.mktime(datetime.datetime.utcnow().timetuple()) + DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler() @@ -348,7 +382,7 @@ class Step(GitRecipesMixin): def DeleteBranch(self, name): for line in self.GitBranch().splitlines(): - if re.match(r".*\s+%s$" % name, line): + if re.match(r"\*?\s*%s$" % re.escape(name), line): msg = "Branch %s exists, do you want to delete it?" % name if self.Confirm(msg): self.GitDeleteBranch(name) @@ -446,6 +480,25 @@ class Step(GitRecipesMixin): return self.GitLog(n=1, format="%H", grep=push_pattern, parent_hash=parent_hash, branch=branch) + def ArrayToVersion(self, prefix): + return ".".join([self[prefix + "major"], + self[prefix + "minor"], + self[prefix + "build"], + self[prefix + "patch"]]) + + def SetVersion(self, version_file, prefix): + output = "" + for line in FileToText(version_file).splitlines(): + if line.startswith("#define MAJOR_VERSION"): + line = re.sub("\d+$", self[prefix + "major"], line) + elif line.startswith("#define MINOR_VERSION"): + line = re.sub("\d+$", self[prefix + "minor"], line) + elif line.startswith("#define BUILD_NUMBER"): + line = re.sub("\d+$", self[prefix + "build"], line) + elif line.startswith("#define PATCH_LEVEL"): + line = re.sub("\d+$", self[prefix + "patch"], line) + output += "%s\n" % line + TextToFile(output, version_file) class UploadStep(Step): MESSAGE = "Upload for code review." diff --git a/deps/v8/tools/push-to-trunk/git_recipes.py b/deps/v8/tools/push-to-trunk/git_recipes.py index 8c1e314d7..6ffb2da83 100644 --- a/deps/v8/tools/push-to-trunk/git_recipes.py +++ b/deps/v8/tools/push-to-trunk/git_recipes.py @@ -68,6 +68,9 @@ class GitRecipesMixin(object): assert name self.Git(MakeArgs(["reset --hard", name])) + def GitStash(self): + self.Git(MakeArgs(["stash"])) + def GitRemotes(self): return map(str.strip, self.Git(MakeArgs(["branch -r"])).splitlines()) @@ -144,7 +147,8 @@ class GitRecipesMixin(object): args.append(Quoted(patch_file)) self.Git(MakeArgs(args)) - def GitUpload(self, reviewer="", author="", force=False): + def GitUpload(self, reviewer="", author="", force=False, cq=False, + bypass_hooks=False): args = ["cl upload --send-mail"] if author: args += ["--email", Quoted(author)] @@ -152,6 +156,10 @@ class GitRecipesMixin(object): args += ["-r", Quoted(reviewer)] if force: args.append("-f") + if cq: + args.append("--use-commit-queue") + if bypass_hooks: + args.append("--bypass-hooks") # TODO(machenbach): Check output in forced mode. Verify that all required # base files were uploaded, if not retry. self.Git(MakeArgs(args), pipe=False) @@ -180,6 +188,9 @@ class GitRecipesMixin(object): def GitSVNFetch(self): self.Git("svn fetch") + def GitSVNRebase(self): + self.Git("svn rebase") + # TODO(machenbach): Unused? Remove. @Strip def GitSVNLog(self): diff --git a/deps/v8/tools/push-to-trunk/push_to_trunk.py b/deps/v8/tools/push-to-trunk/push_to_trunk.py index c317bdc73..56375fe79 100755 --- a/deps/v8/tools/push-to-trunk/push_to_trunk.py +++ b/deps/v8/tools/push-to-trunk/push_to_trunk.py @@ -124,6 +124,20 @@ class DetectLastPush(Step): self["last_push_bleeding_edge"] = last_push_bleeding_edge +# TODO(machenbach): Code similarities with bump_up_version.py. Merge after +# turning this script into a pure git script. +class GetCurrentBleedingEdgeVersion(Step): + MESSAGE = "Get latest bleeding edge version." + + def RunStep(self): + self.GitCheckoutFile(self.Config(VERSION_FILE), "svn/bleeding_edge") + + # Store latest version. + self.ReadAndPersistVersion("latest_") + self["latest_version"] = self.ArrayToVersion("latest_") + print "Bleeding edge version: %s" % self["latest_version"] + + class IncrementVersion(Step): MESSAGE = "Increment version number." @@ -131,11 +145,23 @@ class IncrementVersion(Step): # Retrieve current version from last trunk push. self.GitCheckoutFile(self.Config(VERSION_FILE), self["last_push_trunk"]) self.ReadAndPersistVersion() + self["trunk_version"] = self.ArrayToVersion("") + + if self["latest_build"] == "9999": # pragma: no cover + # If version control on bleeding edge was switched off, just use the last + # trunk version. + self["latest_version"] = self["trunk_version"] + + if SortingKey(self["trunk_version"]) < SortingKey(self["latest_version"]): + # If the version on bleeding_edge is newer than on trunk, use it. + self.GitCheckoutFile(self.Config(VERSION_FILE), "svn/bleeding_edge") + self.ReadAndPersistVersion() if self.Confirm(("Automatically increment BUILD_NUMBER? (Saying 'n' will " "fire up your EDITOR on %s so you can make arbitrary " "changes. When you're done, save the file and exit your " "EDITOR.)" % self.Config(VERSION_FILE))): + text = FileToText(self.Config(VERSION_FILE)) text = MSub(r"(?<=#define BUILD_NUMBER)(?P<space>\s+)\d*$", r"\g<space>%s" % str(int(self["build"]) + 1), @@ -147,6 +173,10 @@ class IncrementVersion(Step): # Variables prefixed with 'new_' contain the new version numbers for the # ongoing trunk push. self.ReadAndPersistVersion("new_") + + # Make sure patch level is 0 in a new push. + self["new_patch"] = "0" + self["version"] = "%s.%s.%s" % (self["new_major"], self["new_minor"], self["new_build"]) @@ -307,20 +337,7 @@ class SetVersion(Step): # The version file has been modified by the patch. Reset it to the version # on trunk and apply the correct version. self.GitCheckoutFile(self.Config(VERSION_FILE), "svn/trunk") - output = "" - for line in FileToText(self.Config(VERSION_FILE)).splitlines(): - if line.startswith("#define MAJOR_VERSION"): - line = re.sub("\d+$", self["new_major"], line) - elif line.startswith("#define MINOR_VERSION"): - line = re.sub("\d+$", self["new_minor"], line) - elif line.startswith("#define BUILD_NUMBER"): - line = re.sub("\d+$", self["new_build"], line) - elif line.startswith("#define PATCH_LEVEL"): - line = re.sub("\d+$", "0", line) - elif line.startswith("#define IS_CANDIDATE_VERSION"): - line = re.sub("\d+$", "0", line) - output += "%s\n" % line - TextToFile(output, self.Config(VERSION_FILE)) + self.SetVersion(self.Config(VERSION_FILE), "new_") class CommitTrunk(Step): @@ -428,6 +445,7 @@ class PushToTrunk(ScriptsBase): FreshBranch, PreparePushRevision, DetectLastPush, + GetCurrentBleedingEdgeVersion, IncrementVersion, PrepareChangeLog, EditChangeLog, diff --git a/deps/v8/tools/push-to-trunk/releases.py b/deps/v8/tools/push-to-trunk/releases.py index 2a22b912e..ff5784496 100755 --- a/deps/v8/tools/push-to-trunk/releases.py +++ b/deps/v8/tools/push-to-trunk/releases.py @@ -52,15 +52,10 @@ DEPS_RE = re.compile(r'^\s*(?:"v8_revision": "' '|"http\:\/\/v8\.googlecode\.com\/svn\/trunk@)' '([0-9]+)".*$', re.M) - -def SortingKey(version): - """Key for sorting version number strings: '3.11' > '3.2.1.1'""" - version_keys = map(int, version.split(".")) - # Fill up to full version numbers to normalize comparison. - while len(version_keys) < 4: - version_keys.append(0) - # Fill digits. - return ".".join(map("{0:03d}".format, version_keys)) +# Expression to pick tag and revision for bleeding edge tags. To be used with +# output of 'svn log'. +BLEEDING_EDGE_TAGS_RE = re.compile( + r"A \/tags\/([^\s]+) \(from \/branches\/bleeding_edge\:(\d+)\)") def SortBranches(branches): @@ -150,24 +145,14 @@ class RetrieveV8Releases(Step): patches = "-%s" % patches return patches - def GetRelease(self, git_hash, branch): - self.ReadAndPersistVersion() - base_version = [self["major"], self["minor"], self["build"]] - version = ".".join(base_version) - body = self.GitLog(n=1, format="%B", git_hash=git_hash) - - patches = "" - if self["patch"] != "0": - version += ".%s" % self["patch"] - patches = self.GetMergedPatches(body) - - title = self.GitLog(n=1, format="%s", git_hash=git_hash) + def GetReleaseDict( + self, git_hash, bleeding_edge_rev, branch, version, patches, cl_body): revision = self.GitSVNFindSVNRev(git_hash) return { # The SVN revision on the branch. "revision": revision, # The SVN revision on bleeding edge (only for newer trunk pushes). - "bleeding_edge": self.GetBleedingEdgeFromPush(title), + "bleeding_edge": bleeding_edge_rev, # The branch name. "branch": branch, # The version for displaying in the form 3.26.3 or 3.26.3.12. @@ -182,14 +167,45 @@ class RetrieveV8Releases(Step): "chromium_branch": "", # Link to the CL on code review. Trunk pushes are not uploaded, so this # field will be populated below with the recent roll CL link. - "review_link": MatchSafe(REVIEW_LINK_RE.search(body)), + "review_link": MatchSafe(REVIEW_LINK_RE.search(cl_body)), # Link to the commit message on google code. "revision_link": ("https://code.google.com/p/v8/source/detail?r=%s" % revision), - }, self["patch"] + } + + def GetRelease(self, git_hash, branch): + self.ReadAndPersistVersion() + base_version = [self["major"], self["minor"], self["build"]] + version = ".".join(base_version) + body = self.GitLog(n=1, format="%B", git_hash=git_hash) + + patches = "" + if self["patch"] != "0": + version += ".%s" % self["patch"] + patches = self.GetMergedPatches(body) + + title = self.GitLog(n=1, format="%s", git_hash=git_hash) + return self.GetReleaseDict( + git_hash, self.GetBleedingEdgeFromPush(title), branch, version, + patches, body), self["patch"] + + def GetReleasesFromBleedingEdge(self): + tag_text = self.SVN("log https://v8.googlecode.com/svn/tags -v --limit 20") + releases = [] + for (tag, revision) in re.findall(BLEEDING_EDGE_TAGS_RE, tag_text): + git_hash = self.GitSVNFindGitHash(revision) + + # Add bleeding edge release. It does not contain patches or a code + # review link, as tags are not uploaded. + releases.append(self.GetReleaseDict( + git_hash, revision, "bleeding_edge", tag, "", "")) + return releases def GetReleasesFromBranch(self, branch): self.GitReset("svn/%s" % branch) + if branch == 'bleeding_edge': + return self.GetReleasesFromBleedingEdge() + releases = [] try: for git_hash in self.GitLog(format="%H").splitlines(): @@ -235,14 +251,16 @@ class RetrieveV8Releases(Step): releases += self.GetReleasesFromBranch(stable) releases += self.GetReleasesFromBranch(beta) releases += self.GetReleasesFromBranch("trunk") + releases += self.GetReleasesFromBranch("bleeding_edge") elif self._options.branch == 'all': # pragma: no cover # Retrieve the full release history. for branch in branches: releases += self.GetReleasesFromBranch(branch) releases += self.GetReleasesFromBranch("trunk") + releases += self.GetReleasesFromBranch("bleeding_edge") else: # pragma: no cover # Retrieve history for a specified branch. - assert self._options.branch in branches + ["trunk"] + assert self._options.branch in branches + ["trunk", "bleeding_edge"] releases += self.GetReleasesFromBranch(self._options.branch) self["releases"] = sorted(releases, diff --git a/deps/v8/tools/push-to-trunk/test_scripts.py b/deps/v8/tools/push-to-trunk/test_scripts.py index bc79cfd5d..82a4d15f2 100644 --- a/deps/v8/tools/push-to-trunk/test_scripts.py +++ b/deps/v8/tools/push-to-trunk/test_scripts.py @@ -35,6 +35,7 @@ import auto_push from auto_push import CheckLastPush from auto_push import SETTINGS_LOCATION import auto_roll +from auto_roll import CLUSTERFUZZ_API_KEY_FILE import common_includes from common_includes import * import merge_to_branch @@ -47,6 +48,11 @@ from chromium_roll import DEPS_FILE from chromium_roll import ChromiumRoll import releases from releases import Releases +import bump_up_version +from bump_up_version import BumpUpVersion +from bump_up_version import LastChangeBailout +from bump_up_version import LKGRVersionUpToDateBailout +from auto_tag import AutoTag TEST_CONFIG = { @@ -66,6 +72,7 @@ TEST_CONFIG = { "/tmp/test-merge-to-branch-tempfile-already-merging", COMMIT_HASHES_FILE: "/tmp/test-merge-to-branch-tempfile-PATCH_COMMIT_HASHES", TEMPORARY_PATCH_FILE: "/tmp/test-merge-to-branch-tempfile-temporary-patch", + CLUSTERFUZZ_API_KEY_FILE: "/tmp/test-fake-cf-api-key", } @@ -350,7 +357,7 @@ class ScriptTest(unittest.TestCase): def RunStep(self, script=PushToTrunk, step_class=Step, args=None): """Convenience wrapper.""" - args = args or ["-m"] + args = args if args is not None else ["-m"] return script(TEST_CONFIG, self, self._state).RunSteps([step_class], args) def GitMock(self, cmd, args="", pipe=True): @@ -384,12 +391,20 @@ class ScriptTest(unittest.TestCase): else: return self._url_mock.Call("readurl", url) + def ReadClusterFuzzAPI(self, api_key, **params): + # TODO(machenbach): Use a mock for this and add a test that stops rolling + # due to clustefuzz results. + return [] + def Sleep(self, seconds): pass def GetDate(self): return "1999-07-31" + def GetUTCStamp(self): + return "100000" + def ExpectGit(self, *args): """Convenience wrapper.""" self._git_mock.Expect(*args) @@ -590,13 +605,19 @@ class ScriptTest(unittest.TestCase): self.assertEquals("New\n Lines", FileToText(TEST_CONFIG[CHANGELOG_ENTRY_FILE])) + # Version on trunk: 3.22.4.0. Version on master (bleeding_edge): 3.22.6. + # Make sure that the increment is 3.22.7.0. def testIncrementVersion(self): TEST_CONFIG[VERSION_FILE] = self.MakeEmptyTempFile() self.WriteFakeVersionFile() self._state["last_push_trunk"] = "hash1" + self._state["latest_build"] = "6" + self._state["latest_version"] = "3.22.6.0" self.ExpectGit([ - Git("checkout -f hash1 -- %s" % TEST_CONFIG[VERSION_FILE], "") + Git("checkout -f hash1 -- %s" % TEST_CONFIG[VERSION_FILE], ""), + Git("checkout -f svn/bleeding_edge -- %s" % TEST_CONFIG[VERSION_FILE], + "", cb=lambda: self.WriteFakeVersionFile(22, 6)), ]) self.ExpectReadline([ @@ -607,7 +628,7 @@ class ScriptTest(unittest.TestCase): self.assertEquals("3", self._state["new_major"]) self.assertEquals("22", self._state["new_minor"]) - self.assertEquals("5", self._state["new_build"]) + self.assertEquals("7", self._state["new_build"]) self.assertEquals("0", self._state["new_patch"]) def _TestSquashCommits(self, change_log, expected_msg): @@ -734,6 +755,8 @@ Performance and stability improvements on all platforms.""", commit) Git("log -1 --format=%s hash2", "Version 3.4.5 (based on bleeding_edge revision r1234)\n"), Git("svn find-rev r1234", "hash3\n"), + Git("checkout -f svn/bleeding_edge -- %s" % TEST_CONFIG[VERSION_FILE], + "", cb=self.WriteFakeVersionFile), Git("checkout -f hash2 -- %s" % TEST_CONFIG[VERSION_FILE], "", cb=self.WriteFakeVersionFile), Git("log --format=%H hash3..push_hash", "rev1\n"), @@ -996,6 +1019,8 @@ deps = { self.assertEquals(1, result) def testAutoRoll(self): + TEST_CONFIG[CLUSTERFUZZ_API_KEY_FILE] = self.MakeEmptyTempFile() + TextToFile("fake key", TEST_CONFIG[CLUSTERFUZZ_API_KEY_FILE]) self.ExpectReadURL([ URL("https://codereview.chromium.org/search", "owner=author%40chromium.org&limit=30&closed=3&format=json", @@ -1142,6 +1167,33 @@ LOG=N MergeToBranch(TEST_CONFIG, self).Run(args) def testReleases(self): + tag_response_text = """ +------------------------------------------------------------------------ +r22631 | author1@chromium.org | 2014-07-28 02:05:29 +0200 (Mon, 28 Jul 2014) +Changed paths: + A /tags/3.28.43 (from /trunk:22630) + +Tagging version 3.28.43 +------------------------------------------------------------------------ +r22629 | author2@chromium.org | 2014-07-26 05:09:29 +0200 (Sat, 26 Jul 2014) +Changed paths: + A /tags/3.28.41 (from /branches/bleeding_edge:22626) + +Tagging version 3.28.41 +------------------------------------------------------------------------ +r22556 | author3@chromium.org | 2014-07-23 13:31:59 +0200 (Wed, 23 Jul 2014) +Changed paths: + A /tags/3.27.34.7 (from /branches/3.27:22555) + +Tagging version 3.27.34.7 +------------------------------------------------------------------------ +r22627 | author4@chromium.org | 2014-07-26 01:39:15 +0200 (Sat, 26 Jul 2014) +Changed paths: + A /tags/3.28.40 (from /branches/bleeding_edge:22624) + +Tagging version 3.28.40 +------------------------------------------------------------------------ +""" json_output = self.MakeEmptyTempFile() csv_output = self.MakeEmptyTempFile() TEST_CONFIG[VERSION_FILE] = self.MakeEmptyTempFile() @@ -1205,6 +1257,15 @@ LOG=N Git("log -1 --format=%ci hash6", ""), Git("checkout -f HEAD -- %s" % TEST_CONFIG[VERSION_FILE], "", cb=ResetVersion(22, 5)), + Git("reset --hard svn/bleeding_edge", ""), + Git("log https://v8.googlecode.com/svn/tags -v --limit 20", + tag_response_text), + Git("svn find-rev r22626", "hash_22626"), + Git("svn find-rev hash_22626", "22626"), + Git("log -1 --format=%ci hash_22626", "01:23"), + Git("svn find-rev r22624", "hash_22624"), + Git("svn find-rev hash_22624", "22624"), + Git("log -1 --format=%ci hash_22624", "02:34"), Git("status -s -uno", ""), Git("checkout -f master", ""), Git("pull", ""), @@ -1235,12 +1296,22 @@ LOG=N Releases(TEST_CONFIG, self).Run(args) # Check expected output. - csv = ("3.22.3,trunk,345,4567,\r\n" + csv = ("3.28.41,bleeding_edge,22626,,\r\n" + "3.28.40,bleeding_edge,22624,,\r\n" + "3.22.3,trunk,345,4567,\r\n" "3.21.2,3.21,123,,\r\n" "3.3.1.1,3.3,234,,12\r\n") self.assertEquals(csv, FileToText(csv_output)) expected_json = [ + {"bleeding_edge": "22626", "patches_merged": "", "version": "3.28.41", + "chromium_revision": "", "branch": "bleeding_edge", "revision": "22626", + "review_link": "", "date": "01:23", "chromium_branch": "", + "revision_link": "https://code.google.com/p/v8/source/detail?r=22626"}, + {"bleeding_edge": "22624", "patches_merged": "", "version": "3.28.40", + "chromium_revision": "", "branch": "bleeding_edge", "revision": "22624", + "review_link": "", "date": "02:34", "chromium_branch": "", + "revision_link": "https://code.google.com/p/v8/source/detail?r=22624"}, {"bleeding_edge": "", "patches_merged": "", "version": "3.22.3", "chromium_revision": "4567", "branch": "trunk", "revision": "345", "review_link": "", "date": "", "chromium_branch": "7", @@ -1257,6 +1328,145 @@ LOG=N self.assertEquals(expected_json, json.loads(FileToText(json_output))) + def testBumpUpVersion(self): + TEST_CONFIG[VERSION_FILE] = self.MakeEmptyTempFile() + self.WriteFakeVersionFile() + + def ResetVersion(minor, build, patch=0): + return lambda: self.WriteFakeVersionFile(minor=minor, + build=build, + patch=patch) + + self.ExpectGit([ + Git("status -s -uno", ""), + Git("checkout -f bleeding_edge", "", cb=ResetVersion(11, 4)), + Git("pull", ""), + Git("branch", ""), + Git("checkout -f bleeding_edge", ""), + Git("log -1 --format=%H", "latest_hash"), + Git("diff --name-only latest_hash latest_hash^", ""), + Git("checkout -f bleeding_edge", ""), + Git("log --format=%H --grep=\"^git-svn-id: [^@]*@12345 [A-Za-z0-9-]*$\"", + "lkgr_hash"), + Git("checkout -b auto-bump-up-version lkgr_hash", ""), + Git("checkout -f bleeding_edge", ""), + Git("branch", ""), + Git("diff --name-only lkgr_hash lkgr_hash^", ""), + Git("checkout -f master", "", cb=ResetVersion(11, 5)), + Git("pull", ""), + Git("checkout -b auto-bump-up-version bleeding_edge", "", + cb=ResetVersion(11, 4)), + Git("commit -am \"[Auto-roll] Bump up version to 3.11.6.0\n\n" + "TBR=author@chromium.org\"", ""), + Git("cl upload --send-mail --email \"author@chromium.org\" -f " + "--bypass-hooks", ""), + Git("cl dcommit -f --bypass-hooks", ""), + Git("checkout -f bleeding_edge", ""), + Git("branch", "auto-bump-up-version\n* bleeding_edge"), + Git("branch -D auto-bump-up-version", ""), + ]) + + self.ExpectReadURL([ + URL("https://v8-status.appspot.com/lkgr", "12345"), + URL("https://v8-status.appspot.com/current?format=json", + "{\"message\": \"Tree is open\"}"), + ]) + + BumpUpVersion(TEST_CONFIG, self).Run(["-a", "author@chromium.org"]) + + def testAutoTag(self): + TEST_CONFIG[VERSION_FILE] = self.MakeEmptyTempFile() + self.WriteFakeVersionFile() + + def ResetVersion(minor, build, patch=0): + return lambda: self.WriteFakeVersionFile(minor=minor, + build=build, + patch=patch) + + self.ExpectGit([ + Git("status -s -uno", ""), + Git("status -s -b -uno", "## some_branch\n"), + Git("svn fetch", ""), + Git("branch", " branch1\n* branch2\n"), + Git("checkout -f master", ""), + Git("svn rebase", ""), + Git("checkout -b %s" % TEST_CONFIG[BRANCHNAME], "", + cb=ResetVersion(4, 5)), + Git("branch -r", "svn/tags/3.4.2\nsvn/tags/3.2.1.0\nsvn/branches/3.4"), + Git("log --format=%H --grep=\"\\[Auto\\-roll\\] Bump up version to\"", + "hash125\nhash118\nhash111\nhash101"), + Git("checkout -f hash125 -- %s" % TEST_CONFIG[VERSION_FILE], "", + cb=ResetVersion(4, 4)), + Git("checkout -f HEAD -- %s" % TEST_CONFIG[VERSION_FILE], "", + cb=ResetVersion(4, 5)), + Git("checkout -f hash118 -- %s" % TEST_CONFIG[VERSION_FILE], "", + cb=ResetVersion(4, 3)), + Git("checkout -f HEAD -- %s" % TEST_CONFIG[VERSION_FILE], "", + cb=ResetVersion(4, 5)), + Git("checkout -f hash111 -- %s" % TEST_CONFIG[VERSION_FILE], "", + cb=ResetVersion(4, 2)), + Git("checkout -f HEAD -- %s" % TEST_CONFIG[VERSION_FILE], "", + cb=ResetVersion(4, 5)), + Git("svn find-rev hash118", "118"), + Git("svn find-rev hash125", "125"), + Git("svn find-rev r123", "hash123"), + Git("log -1 --format=%at hash123", "1"), + Git("reset --hard hash123", ""), + Git("svn tag 3.4.3 -m \"Tagging version 3.4.3\"", ""), + Git("checkout -f some_branch", ""), + Git("branch -D %s" % TEST_CONFIG[BRANCHNAME], ""), + ]) + + self.ExpectReadURL([ + URL("https://v8-status.appspot.com/revisions?format=json", + "[{\"revision\": \"126\", \"status\": true}," + "{\"revision\": \"123\", \"status\": true}," + "{\"revision\": \"112\", \"status\": true}]"), + ]) + + AutoTag(TEST_CONFIG, self).Run(["-a", "author@chromium.org"]) + + # Test that we bail out if the last change was a version change. + def testBumpUpVersionBailout1(self): + TEST_CONFIG[VERSION_FILE] = self.MakeEmptyTempFile() + self._state["latest"] = "latest_hash" + + self.ExpectGit([ + Git("diff --name-only latest_hash latest_hash^", + TEST_CONFIG[VERSION_FILE]), + ]) + + self.assertEquals(1, + self.RunStep(BumpUpVersion, LastChangeBailout, ["--dry_run"])) + + # Test that we bail out if the lkgr was a version change. + def testBumpUpVersionBailout2(self): + TEST_CONFIG[VERSION_FILE] = self.MakeEmptyTempFile() + self._state["lkgr"] = "lkgr_hash" + + self.ExpectGit([ + Git("diff --name-only lkgr_hash lkgr_hash^", TEST_CONFIG[VERSION_FILE]), + ]) + + self.assertEquals(1, + self.RunStep(BumpUpVersion, LKGRVersionUpToDateBailout, ["--dry_run"])) + + # Test that we bail out if the last version is already newer than the lkgr's + # version. + def testBumpUpVersionBailout3(self): + TEST_CONFIG[VERSION_FILE] = self.MakeEmptyTempFile() + self._state["lkgr"] = "lkgr_hash" + self._state["lkgr_version"] = "3.22.4.0" + self._state["latest_version"] = "3.22.5.0" + + self.ExpectGit([ + Git("diff --name-only lkgr_hash lkgr_hash^", ""), + ]) + + self.assertEquals(1, + self.RunStep(BumpUpVersion, LKGRVersionUpToDateBailout, ["--dry_run"])) + + class SystemTest(unittest.TestCase): def testReload(self): step = MakeStep(step_class=PrepareChangeLog, number=0, state={}, config={}, |