# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from optparse import make_option from webkitpy.tool import steps from webkitpy.common.checkout.commitinfo import CommitInfo from webkitpy.common.config.committers import CommitterList import webkitpy.common.config.urls as config_urls from webkitpy.common.net.buildbot import BuildBot from webkitpy.common.net.regressionwindow import RegressionWindow from webkitpy.common.system.crashlogs import CrashLogs from webkitpy.common.system.user import User from webkitpy.tool.grammar import pluralize from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand from webkitpy.common.system.deprecated_logging import log from webkitpy.layout_tests import port class SuggestReviewers(AbstractDeclarativeCommand): name = "suggest-reviewers" help_text = "Suggest reviewers for a patch based on recent changes to the modified files." def __init__(self): options = [ steps.Options.git_commit, ] AbstractDeclarativeCommand.__init__(self, options=options) def execute(self, options, args, tool): reviewers = tool.checkout().suggested_reviewers(options.git_commit) print "\n".join([reviewer.full_name for reviewer in reviewers]) class BugsToCommit(AbstractDeclarativeCommand): name = "bugs-to-commit" help_text = "List bugs in the commit-queue" def execute(self, options, args, tool): # FIXME: This command is poorly named. It's fetching the commit-queue list here. The name implies it's fetching pending-commit (all r+'d patches). bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue() for bug_id in bug_ids: print "%s" % bug_id class PatchesInCommitQueue(AbstractDeclarativeCommand): name = "patches-in-commit-queue" help_text = "List patches in the commit-queue" def execute(self, options, args, tool): patches = tool.bugs.queries.fetch_patches_from_commit_queue() log("Patches in commit queue:") for patch in patches: print patch.url() class PatchesToCommitQueue(AbstractDeclarativeCommand): name = "patches-to-commit-queue" help_text = "Patches which should be added to the commit queue" def __init__(self): options = [ make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"), ] AbstractDeclarativeCommand.__init__(self, options=options) @staticmethod def _needs_commit_queue(patch): if patch.commit_queue() == "+": # If it's already cq+, ignore the patch. log("%s already has cq=%s" % (patch.id(), patch.commit_queue())) return False # We only need to worry about patches from contributers who are not yet committers. committer_record = CommitterList().committer_by_email(patch.attacher_email()) if committer_record: log("%s committer = %s" % (patch.id(), committer_record)) return not committer_record def execute(self, options, args, tool): patches = tool.bugs.queries.fetch_patches_from_pending_commit_list() patches_needing_cq = filter(self._needs_commit_queue, patches) if options.bugs: bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq) bugs_needing_cq = sorted(set(bugs_needing_cq)) for bug_id in bugs_needing_cq: print "%s" % tool.bugs.bug_url_for_bug_id(bug_id) else: for patch in patches_needing_cq: print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit") class PatchesToReview(AbstractDeclarativeCommand): name = "patches-to-review" help_text = "List patches that are pending review" def execute(self, options, args, tool): patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue() log("Patches pending review:") for patch_id in patch_ids: print patch_id class LastGreenRevision(AbstractDeclarativeCommand): name = "last-green-revision" help_text = "Prints the last known good revision" def execute(self, options, args, tool): print self._tool.buildbot.last_green_revision() class WhatBroke(AbstractDeclarativeCommand): name = "what-broke" help_text = "Print failing buildbots (%s) and what revisions broke them" % config_urls.buildbot_url def _print_builder_line(self, builder_name, max_name_width, status_message): print "%s : %s" % (builder_name.ljust(max_name_width), status_message) def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True): builder = self._tool.buildbot.builder_with_name(builder_status["name"]) red_build = builder.build(builder_status["build_number"]) regression_window = builder.find_regression_window(red_build) if not regression_window.failing_build(): self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)") return if not regression_window.build_before_failure(): self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % regression_window.failing_build().revision()) return revisions = regression_window.revisions() first_failure_message = "" if (regression_window.failing_build() == builder.build(builder_status["build_number"])): first_failure_message = " FIRST FAILURE, possibly a flaky test" self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (revisions, first_failure_message)) for revision in revisions: commit_info = self._tool.checkout().commit_info_for_revision(revision) if commit_info: print commit_info.blame_string(self._tool.bugs) else: print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision def execute(self, options, args, tool): builder_statuses = tool.buildbot.builder_statuses() longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses))) failing_builders = 0 for builder_status in builder_statuses: # If the builder is green, print OK, exit. if builder_status["is_green"]: continue self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name) failing_builders += 1 if failing_builders: print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses))) else: print "All builders are passing!" class ResultsFor(AbstractDeclarativeCommand): name = "results-for" help_text = "Print a list of failures for the passed revision from bots on %s" % config_urls.buildbot_url argument_names = "REVISION" def _print_layout_test_results(self, results): if not results: print " No results." return for title, files in results.parsed_results().items(): print " %s" % title for filename in files: print " %s" % filename def execute(self, options, args, tool): builders = self._tool.buildbot.builders() for builder in builders: print "%s:" % builder.name() build = builder.build_for_revision(args[0], allow_failed_lookups=True) self._print_layout_test_results(build.layout_test_results()) class FailureReason(AbstractDeclarativeCommand): name = "failure-reason" help_text = "Lists revisions where individual test failures started at %s" % config_urls.buildbot_url def _blame_line_for_revision(self, revision): try: commit_info = self._tool.checkout().commit_info_for_revision(revision) except Exception, e: return "FAILED to fetch CommitInfo for r%s, exception: %s" % (revision, e) if not commit_info: return "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision return commit_info.blame_string(self._tool.bugs) def _print_blame_information_for_transition(self, regression_window, failing_tests): red_build = regression_window.failing_build() print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests) print "Suspect revisions:" for revision in regression_window.revisions(): print self._blame_line_for_revision(revision) def _explain_failures_for_builder(self, builder, start_revision): print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision) revision_to_test = start_revision build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) layout_test_results = build.layout_test_results() if not layout_test_results: # FIXME: This could be made more user friendly. print "Failed to load layout test results from %s; can't continue. (start revision = r%s)" % (build.results_url(), start_revision) return 1 results_to_explain = set(layout_test_results.failing_tests()) last_build_with_results = build print "Starting at %s" % revision_to_test while results_to_explain: revision_to_test -= 1 new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) if not new_build: print "No build for %s" % revision_to_test continue build = new_build latest_results = build.layout_test_results() if not latest_results: print "No results build %s (r%s)" % (build._number, build.revision()) continue failures = set(latest_results.failing_tests()) if len(failures) >= 20: # FIXME: We may need to move this logic into the LayoutTestResults class. # The buildbot stops runs after 20 failures so we don't have full results to work with here. print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision()) continue fixed_results = results_to_explain - failures if not fixed_results: print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures)) last_build_with_results = build continue regression_window = RegressionWindow(build, last_build_with_results) self._print_blame_information_for_transition(regression_window, fixed_results) last_build_with_results = build results_to_explain -= fixed_results if results_to_explain: print "Failed to explain failures: %s" % results_to_explain return 1 print "Explained all results for %s" % builder.name() return 0 def _builder_to_explain(self): builder_statuses = self._tool.buildbot.builder_statuses() red_statuses = [status for status in builder_statuses if not status["is_green"]] print "%s failing" % (pluralize("builder", len(red_statuses))) builder_choices = [status["name"] for status in red_statuses] # We could offer an "All" choice here. chosen_name = self._tool.user.prompt_with_list("Which builder to diagnose:", builder_choices) # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object. for status in red_statuses: if status["name"] == chosen_name: return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"]) def execute(self, options, args, tool): (builder, latest_revision) = self._builder_to_explain() start_revision = self._tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision if not start_revision: print "Revision required." return 1 return self._explain_failures_for_builder(builder, start_revision=int(start_revision)) class FindFlakyTests(AbstractDeclarativeCommand): name = "find-flaky-tests" help_text = "Lists tests that often fail for a single build at %s" % config_urls.buildbot_url def _find_failures(self, builder, revision): build = builder.build_for_revision(revision, allow_failed_lookups=True) if not build: print "No build for %s" % revision return (None, None) results = build.layout_test_results() if not results: print "No results build %s (r%s)" % (build._number, build.revision()) return (None, None) failures = set(results.failing_tests()) if len(failures) >= 20: # FIXME: We may need to move this logic into the LayoutTestResults class. # The buildbot stops runs after 20 failures so we don't have full results to work with here. print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision()) return (None, None) return (build, failures) def _increment_statistics(self, flaky_tests, flaky_test_statistics): for test in flaky_tests: count = flaky_test_statistics.get(test, 0) flaky_test_statistics[test] = count + 1 def _print_statistics(self, statistics): print "=== Results ===" print "Occurances Test name" for value, key in sorted([(value, key) for key, value in statistics.items()]): print "%10d %s" % (value, key) def _walk_backwards_from(self, builder, start_revision, limit): flaky_test_statistics = {} all_previous_failures = set([]) one_time_previous_failures = set([]) previous_build = None for i in range(limit): revision = start_revision - i print "Analyzing %s ... " % revision, (build, failures) = self._find_failures(builder, revision) if failures == None: # Notice that we don't loop on the empty set! continue print "has %s failures" % len(failures) flaky_tests = one_time_previous_failures - failures if flaky_tests: print "Flaky tests: %s %s" % (sorted(flaky_tests), previous_build.results_url()) self._increment_statistics(flaky_tests, flaky_test_statistics) one_time_previous_failures = failures - all_previous_failures all_previous_failures = failures previous_build = build self._print_statistics(flaky_test_statistics) def _builder_to_analyze(self): statuses = self._tool.buildbot.builder_statuses() choices = [status["name"] for status in statuses] chosen_name = self._tool.user.prompt_with_list("Which builder to analyze:", choices) for status in statuses: if status["name"] == chosen_name: return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"]) def execute(self, options, args, tool): (builder, latest_revision) = self._builder_to_analyze() limit = self._tool.user.prompt("How many revisions to look through? [10000] ") or 10000 return self._walk_backwards_from(builder, latest_revision, limit=int(limit)) class TreeStatus(AbstractDeclarativeCommand): name = "tree-status" help_text = "Print the status of the %s buildbots" % config_urls.buildbot_url long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder and displayes the status of each builder.""" def execute(self, options, args, tool): for builder in tool.buildbot.builder_statuses(): status_string = "ok" if builder["is_green"] else "FAIL" print "%s : %s" % (status_string.ljust(4), builder["name"]) class CrashLog(AbstractDeclarativeCommand): name = "crash-log" argument_names = "PROCESS_NAME" def execute(self, options, args, tool): crash_logs = CrashLogs(tool.filesystem) print crash_logs.find_newest_log(args[0]) class SkippedPorts(AbstractDeclarativeCommand): name = "skipped-ports" help_text = "Print the list of ports skipping the given layout test(s)" long_help = """Scans the the Skipped file of each port and figure out what ports are skipping the test(s). Categories are taken in account too.""" argument_names = "TEST_NAME" def execute(self, options, args, tool): results = dict([(test_name, []) for test_name in args]) for port_name in tool.port_factory.all_port_names(): port_object = tool.port_factory.get(port_name) for test_name in args: if port_object.skips_layout_test(test_name): results[test_name].append(port_name) for test_name, ports in results.iteritems(): if ports: print "Ports skipping test %r: %s" % (test_name, ', '.join(ports)) else: print "Test %r is not skipped by any port." % test_name