path: root/Tools/Scripts/webkitpy/tool/commands/queries.py

# Copyright (c) 2009 Google Inc. All rights reserved.
# Copyright (c) 2009 Apple Inc. All rights reserved.
# Copyright (c) 2012 Intel Corporation. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import fnmatch
import re

from datetime import datetime
from optparse import make_option

from webkitpy.tool import steps

from webkitpy.common.checkout.commitinfo import CommitInfo
from webkitpy.common.config.committers import CommitterList
import webkitpy.common.config.urls as config_urls
from webkitpy.common.net.buildbot import BuildBot
from webkitpy.common.net.regressionwindow import RegressionWindow
from webkitpy.common.system.crashlogs import CrashLogs
from webkitpy.common.system.user import User
from webkitpy.tool.grammar import pluralize
from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
from webkitpy.common.system.deprecated_logging import log
from webkitpy.layout_tests.models.test_expectations import TestExpectations
from webkitpy.layout_tests.port import platform_options, configuration_options


class SuggestReviewers(AbstractDeclarativeCommand):
    name = "suggest-reviewers"
    help_text = "Suggest reviewers for a patch based on recent changes to the modified files."

    def __init__(self):
        options = [
            steps.Options.git_commit,
        ]
        AbstractDeclarativeCommand.__init__(self, options=options)

    def execute(self, options, args, tool):
        reviewers = tool.checkout().suggested_reviewers(options.git_commit)
        print "\n".join([reviewer.full_name for reviewer in reviewers])


class BugsToCommit(AbstractDeclarativeCommand):
    name = "bugs-to-commit"
    help_text = "List bugs in the commit-queue"

    def execute(self, options, args, tool):
        # FIXME: This command is poorly named.  It's fetching the commit-queue list here.  The name implies it's fetching pending-commit (all r+'d patches).
        bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue()
        for bug_id in bug_ids:
            print "%s" % bug_id


class PatchesInCommitQueue(AbstractDeclarativeCommand):
    name = "patches-in-commit-queue"
    help_text = "List patches in the commit-queue"

    def execute(self, options, args, tool):
        patches = tool.bugs.queries.fetch_patches_from_commit_queue()
        log("Patches in commit queue:")
        for patch in patches:
            print patch.url()


class PatchesToCommitQueue(AbstractDeclarativeCommand):
    name = "patches-to-commit-queue"
    help_text = "Patches which should be added to the commit queue"
    def __init__(self):
        options = [
            make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"),
        ]
        AbstractDeclarativeCommand.__init__(self, options=options)

    @staticmethod
    def _needs_commit_queue(patch):
        if patch.commit_queue() == "+": # If it's already cq+, ignore the patch.
            log("%s already has cq=%s" % (patch.id(), patch.commit_queue()))
            return False

        # We only need to worry about patches from contributers who are not yet committers.
        committer_record = CommitterList().committer_by_email(patch.attacher_email())
        if committer_record:
            log("%s committer = %s" % (patch.id(), committer_record))
        return not committer_record

    def execute(self, options, args, tool):
        patches = tool.bugs.queries.fetch_patches_from_pending_commit_list()
        patches_needing_cq = filter(self._needs_commit_queue, patches)
        if options.bugs:
            bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq)
            bugs_needing_cq = sorted(set(bugs_needing_cq))
            for bug_id in bugs_needing_cq:
                print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
        else:
            for patch in patches_needing_cq:
                print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit")


class PatchesToReview(AbstractDeclarativeCommand):
    name = "patches-to-review"
    help_text = "List bugs which have attachments pending review"

    def __init__(self):
        options = [
            make_option("--all", action="store_true",
                        help="Show all bugs regardless of who is on CC (it might take a while)"),
            make_option("--include-cq-denied", action="store_true",
                        help="By default, r? patches with cq- are omitted unless this option is set"),
            make_option("--cc-email",
                        help="Specifies the email on the CC field (defaults to your bugzilla login email)"),
        ]
        AbstractDeclarativeCommand.__init__(self, options=options)

    def _print_report(self, report, cc_email, print_all):
        if print_all:
            print "Bugs with attachments pending review:"
        else:
            print "Bugs with attachments pending review that has %s in the CC list:" % cc_email

        print "http://webkit.org/b/bugid   Description (age in days)"
        for row in report:
            print "%s (%d)" % (row[1], row[0])

        print "Total: %d" % len(report)

    def _generate_report(self, bugs, include_cq_denied):
        report = []

        for bug in bugs:
            patch = bug.unreviewed_patches()[-1]

            if not include_cq_denied and patch.commit_queue() == "-":
                continue

            age_in_days = (datetime.today() - patch.attach_date()).days
            report.append((age_in_days, "http://webkit.org/b/%-7s %s" % (bug.id(), bug.title())))

        report.sort()
        return report

    def execute(self, options, args, tool):
        tool.bugs.authenticate()

        cc_email = options.cc_email
        if not cc_email and not options.all:
            cc_email = tool.bugs.username

        bugs = tool.bugs.queries.fetch_bugs_from_review_queue(cc_email=cc_email)
        report = self._generate_report(bugs, options.include_cq_denied)
        self._print_report(report, cc_email, options.all)

class WhatBroke(AbstractDeclarativeCommand):
    name = "what-broke"
    help_text = "Print failing buildbots (%s) and what revisions broke them" % config_urls.buildbot_url

    def _print_builder_line(self, builder_name, max_name_width, status_message):
        print "%s : %s" % (builder_name.ljust(max_name_width), status_message)

    def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True):
        builder = self._tool.buildbot.builder_with_name(builder_status["name"])
        red_build = builder.build(builder_status["build_number"])
        regression_window = builder.find_regression_window(red_build)
        if not regression_window.failing_build():
            self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)")
            return
        if not regression_window.build_before_failure():
            self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % regression_window.failing_build().revision())
            return

        revisions = regression_window.revisions()
        first_failure_message = ""
        if (regression_window.failing_build() == builder.build(builder_status["build_number"])):
            first_failure_message = " FIRST FAILURE, possibly a flaky test"
        self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (revisions, first_failure_message))
        for revision in revisions:
            commit_info = self._tool.checkout().commit_info_for_revision(revision)
            if commit_info:
                print commit_info.blame_string(self._tool.bugs)
            else:
                print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision

    def execute(self, options, args, tool):
        builder_statuses = tool.buildbot.builder_statuses()
        longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses)))
        failing_builders = 0
        for builder_status in builder_statuses:
            # If the builder is green, print OK, exit.
            if builder_status["is_green"]:
                continue
            self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name)
            failing_builders += 1
        if failing_builders:
            print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses)))
        else:
            print "All builders are passing!"


class ResultsFor(AbstractDeclarativeCommand):
    name = "results-for"
    help_text = "Print a list of failures for the passed revision from bots on %s" % config_urls.buildbot_url
    argument_names = "REVISION"

    def _print_layout_test_results(self, results):
        if not results:
            print " No results."
            return
        for title, files in results.parsed_results().items():
            print " %s" % title
            for filename in files:
                print "  %s" % filename

    def execute(self, options, args, tool):
        builders = self._tool.buildbot.builders()
        for builder in builders:
            print "%s:" % builder.name()
            build = builder.build_for_revision(args[0], allow_failed_lookups=True)
            self._print_layout_test_results(build.layout_test_results())


class FailureReason(AbstractDeclarativeCommand):
    name = "failure-reason"
    help_text = "Lists revisions where individual test failures started at %s" % config_urls.buildbot_url

    def _blame_line_for_revision(self, revision):
        try:
            commit_info = self._tool.checkout().commit_info_for_revision(revision)
        except Exception, e:
            return "FAILED to fetch CommitInfo for r%s, exception: %s" % (revision, e)
        if not commit_info:
            return "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision
        return commit_info.blame_string(self._tool.bugs)

    def _print_blame_information_for_transition(self, regression_window, failing_tests):
        red_build = regression_window.failing_build()
        print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests)
        print "Suspect revisions:"
        for revision in regression_window.revisions():
            print self._blame_line_for_revision(revision)

    def _explain_failures_for_builder(self, builder, start_revision):
        print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision)
        revision_to_test = start_revision
        build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
        layout_test_results = build.layout_test_results()
        if not layout_test_results:
            # FIXME: This could be made more user friendly.
            print "Failed to load layout test results from %s; can't continue. (start revision = r%s)" % (build.results_url(), start_revision)
            return 1

        results_to_explain = set(layout_test_results.failing_tests())
        last_build_with_results = build
        print "Starting at %s" % revision_to_test
        while results_to_explain:
            revision_to_test -= 1
            new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
            if not new_build:
                print "No build for %s" % revision_to_test
                continue
            build = new_build
            latest_results = build.layout_test_results()
            if not latest_results:
                print "No results build %s (r%s)" % (build._number, build.revision())
                continue
            failures = set(latest_results.failing_tests())
            if len(failures) >= 20:
                # FIXME: We may need to move this logic into the LayoutTestResults class.
                # The buildbot stops runs after 20 failures so we don't have full results to work with here.
                print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
                continue
            fixed_results = results_to_explain - failures
            if not fixed_results:
                print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures))
                last_build_with_results = build
                continue
            regression_window = RegressionWindow(build, last_build_with_results)
            self._print_blame_information_for_transition(regression_window, fixed_results)
            last_build_with_results = build
            results_to_explain -= fixed_results
        if results_to_explain:
            print "Failed to explain failures: %s" % results_to_explain
            return 1
        print "Explained all results for %s" % builder.name()
        return 0

    def _builder_to_explain(self):
        builder_statuses = self._tool.buildbot.builder_statuses()
        red_statuses = [status for status in builder_statuses if not status["is_green"]]
        print "%s failing" % (pluralize("builder", len(red_statuses)))
        builder_choices = [status["name"] for status in red_statuses]
        # We could offer an "All" choice here.
        chosen_name = self._tool.user.prompt_with_list("Which builder to diagnose:", builder_choices)
        # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
        for status in red_statuses:
            if status["name"] == chosen_name:
                return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])

    def execute(self, options, args, tool):
        (builder, latest_revision) = self._builder_to_explain()
        start_revision = self._tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision
        if not start_revision:
            print "Revision required."
            return 1
        return self._explain_failures_for_builder(builder, start_revision=int(start_revision))


class FindFlakyTests(AbstractDeclarativeCommand):
    name = "find-flaky-tests"
    help_text = "Lists tests that often fail for a single build at %s" % config_urls.buildbot_url

    def _find_failures(self, builder, revision):
        build = builder.build_for_revision(revision, allow_failed_lookups=True)
        if not build:
            print "No build for %s" % revision
            return (None, None)
        results = build.layout_test_results()
        if not results:
            print "No results build %s (r%s)" % (build._number, build.revision())
            return (None, None)
        failures = set(results.failing_tests())
        if len(failures) >= 20:
            # FIXME: We may need to move this logic into the LayoutTestResults class.
            # The buildbot stops runs after 20 failures so we don't have full results to work with here.
            print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
            return (None, None)
        return (build, failures)

    def _increment_statistics(self, flaky_tests, flaky_test_statistics):
        for test in flaky_tests:
            count = flaky_test_statistics.get(test, 0)
            flaky_test_statistics[test] = count + 1

    def _print_statistics(self, statistics):
        print "=== Results ==="
        print "Occurances Test name"
        for value, key in sorted([(value, key) for key, value in statistics.items()]):
            print "%10d %s" % (value, key)

    def _walk_backwards_from(self, builder, start_revision, limit):
        flaky_test_statistics = {}
        all_previous_failures = set([])
        one_time_previous_failures = set([])
        previous_build = None
        for i in range(limit):
            revision = start_revision - i
            print "Analyzing %s ... " % revision,
            (build, failures) = self._find_failures(builder, revision)
            if failures == None:
                # Notice that we don't loop on the empty set!
                continue
            print "has %s failures" % len(failures)
            flaky_tests = one_time_previous_failures - failures
            if flaky_tests:
                print "Flaky tests: %s %s" % (sorted(flaky_tests),
                                              previous_build.results_url())
            self._increment_statistics(flaky_tests, flaky_test_statistics)
            one_time_previous_failures = failures - all_previous_failures
            all_previous_failures = failures
            previous_build = build
        self._print_statistics(flaky_test_statistics)

    def _builder_to_analyze(self):
        statuses = self._tool.buildbot.builder_statuses()
        choices = [status["name"] for status in statuses]
        chosen_name = self._tool.user.prompt_with_list("Which builder to analyze:", choices)
        for status in statuses:
            if status["name"] == chosen_name:
                return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])

    def execute(self, options, args, tool):
        (builder, latest_revision) = self._builder_to_analyze()
        limit = self._tool.user.prompt("How many revisions to look through? [10000] ") or 10000
        return self._walk_backwards_from(builder, latest_revision, limit=int(limit))


class TreeStatus(AbstractDeclarativeCommand):
    name = "tree-status"
    help_text = "Print the status of the %s buildbots" % config_urls.buildbot_url
    long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder
and displayes the status of each builder."""

    def execute(self, options, args, tool):
        for builder in tool.buildbot.builder_statuses():
            status_string = "ok" if builder["is_green"] else "FAIL"
            print "%s : %s" % (status_string.ljust(4), builder["name"])


class CrashLog(AbstractDeclarativeCommand):
    name = "crash-log"
    help_text = "Print the newest crash log for the given process"
    long_help = """Finds the newest crash log matching the given process name
and PID and prints it to stdout."""
    argument_names = "PROCESS_NAME [PID]"

    def execute(self, options, args, tool):
        crash_logs = CrashLogs(tool)
        pid = None
        if len(args) > 1:
            pid = int(args[1])
        print crash_logs.find_newest_log(args[0], pid)


class PrintExpectations(AbstractDeclarativeCommand):
    name = 'print-expectations'
    help_text = 'Print the expected result for the given test(s) on the given port(s)'

    def __init__(self):
        options = [
            make_option('--all', action='store_true', default=False,
                        help='display the expectations for *all* tests'),
            make_option('-x', '--exclude-keyword', action='append', default=[],
                        help='limit to tests not matching the given keyword (for example, "skip", "slow", or "crash". May specify multiple times'),
            make_option('-i', '--include-keyword', action='append', default=[],
                        help='limit to tests with the given keyword (for example, "skip", "slow", or "crash". May specify multiple times'),
            make_option('--csv', action='store_true', default=False,
                        help='Print a CSV-style report that includes the port name, modifiers, tests, and expectations'),
            make_option('-f', '--full', action='store_true', default=False,
                        help='Print a full TestExpectations-style line for every match'),
            make_option('--paths', action='store_true', default=False,
                        help='display the paths for all applicable expectation files'),
        ] + platform_options(use_globs=True)

        AbstractDeclarativeCommand.__init__(self, options=options)
        self._expectation_models = {}

    def execute(self, options, args, tool):
        if not options.paths and not args and not options.all:
            print "You must either specify one or more test paths or --all."
            return

        if options.platform:
            port_names = fnmatch.filter(tool.port_factory.all_port_names(), options.platform)
            if not port_names:
                default_port = tool.port_factory.get(options.platform)
                if default_port:
                    port_names = [default_port.name()]
                else:
                    print "No port names match '%s'" % options.platform
                    return
            else:
                default_port = tool.port_factory.get(port_names[0])
        else:
            default_port = tool.port_factory.get(options=options)
            port_names = [default_port.name()]

        if options.paths:
            files = default_port.expectations_files()
            layout_tests_dir = default_port.layout_tests_dir()
            for file in files:
                if file.startswith(layout_tests_dir):
                    file = file.replace(layout_tests_dir, 'LayoutTests')
                print file
            return

        tests = default_port.tests(args)
        for port_name in port_names:
            model = self._model(options, port_name, tests)
            tests_to_print = self._filter_tests(options, model, tests)
            lines = [model.get_expectation_line(test) for test in sorted(tests_to_print)]
            if port_name != port_names[0]:
                print
            print '\n'.join(self._format_lines(options, port_name, lines))

    def _filter_tests(self, options, model, tests):
        filtered_tests = set()
        if options.include_keyword:
            for keyword in options.include_keyword:
                filtered_tests.update(model.get_test_set_for_keyword(keyword))
        else:
            filtered_tests = tests

        for keyword in options.exclude_keyword:
            filtered_tests.difference_update(model.get_test_set_for_keyword(keyword))
        return filtered_tests

    def _format_lines(self, options, port_name, lines):
        output = []
        if options.csv:
            for line in lines:
                output.append("%s,%s" % (port_name, line.to_csv()))
        elif lines:
            include_modifiers = options.full
            include_expectations = options.full or len(options.include_keyword) != 1 or len(options.exclude_keyword)
            output.append("// For %s" % port_name)
            for line in lines:
                output.append("%s" % line.to_string(None, include_modifiers, include_expectations, include_comment=False))
        return output

    def _model(self, options, port_name, tests):
        port = self._tool.port_factory.get(port_name, options)
        expectations_path = port.path_to_test_expectations_file()
        if not expectations_path in self._expectation_models:
            self._expectation_models[expectations_path] = TestExpectations(port, tests).model()
        return self._expectation_models[expectations_path]


class PrintBaselines(AbstractDeclarativeCommand):
    name = 'print-baselines'
    help_text = 'Prints the baseline locations for given test(s) on the given port(s)'

    def __init__(self):
        options = [
            make_option('--all', action='store_true', default=False,
                        help='display the baselines for *all* tests'),
            make_option('--csv', action='store_true', default=False,
                        help='Print a CSV-style report that includes the port name, test_name, test platform, baseline type, baseline location, and baseline platform'),
            make_option('--include-virtual-tests', action='store_true',
                        help='Include virtual tests'),
        ] + platform_options(use_globs=True)
        AbstractDeclarativeCommand.__init__(self, options=options)
        self._platform_regexp = re.compile('platform/([^\/]+)/(.+)')

    def execute(self, options, args, tool):
        if not args and not options.all:
            print "You must either specify one or more test paths or --all."
            return

        default_port = tool.port_factory.get()
        if options.platform:
            port_names = fnmatch.filter(tool.port_factory.all_port_names(), options.platform)
            if not port_names:
                print "No port names match '%s'" % options.platform
        else:
            port_names = [default_port.name()]

        if options.include_virtual_tests:
            tests = sorted(default_port.tests(args))
        else:
            # FIXME: make real_tests() a public method.
            tests = sorted(default_port._real_tests(args))

        for port_name in port_names:
            if port_name != port_names[0]:
                print
            if not options.csv:
                print "// For %s" % port_name
            port = tool.port_factory.get(port_name)
            for test_name in tests:
                self._print_baselines(options, port_name, test_name, port.expected_baselines_by_extension(test_name))

    def _print_baselines(self, options, port_name, test_name, baselines):
        for extension in sorted(baselines.keys()):
            baseline_location = baselines[extension]
            if baseline_location:
                if options.csv:
                    print "%s,%s,%s,%s,%s,%s" % (port_name, test_name, self._platform_for_path(test_name),
                                                 extension[1:], baseline_location, self._platform_for_path(baseline_location))
                else:
                    print baseline_location

    def _platform_for_path(self, relpath):
        platform_matchobj = self._platform_regexp.match(relpath)
        if platform_matchobj:
            return platform_matchobj.group(1)
        return None