| # Copyright (c) 2011 Google Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import os |
| import optparse |
| from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand |
| from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory |
| from webkitpy.layout_tests.models.test_expectations import TestExpectationParser, TestExpectationsModel, TestExpectations |
| from webkitpy.layout_tests.port import builders |
| from webkitpy.common.net import sheriff_calendar |
| |
| |
| class FlakyTests(AbstractDeclarativeCommand): |
| name = "update-flaky-tests" |
| help_text = "Update FlakyTests file from the flakiness dashboard" |
| show_in_main_help = True |
| |
| ALWAYS_CC = [ |
| 'ojan@chromium.org', |
| 'dpranke@chromium.org', |
| 'eseidel@chromium.org', |
| ] |
| |
| COMMIT_MESSAGE = ( |
| 'Update FlakyTests to match current flakiness dashboard results\n\n' |
| 'Automatically generated using:\n' |
| 'webkit-patch update-flaky-tests\n\n' |
| 'R=%s\n') |
| |
| FLAKY_TEST_CONTENTS = ( |
| '# This file is generated by webkit-patch update-flaky-tests from the flakiness dashboard data.\n' |
| '# Manual changes will be overwritten.\n\n' |
| '%s\n') |
| |
| def __init__(self): |
| options = [ |
| optparse.make_option('--upload', action='store_true', |
| help='upload the changed FlakyTest file for review'), |
| optparse.make_option('--reviewers', action='store', |
| help='comma-separated list of reviewers, defaults to blink gardeners'), |
| ] |
| AbstractDeclarativeCommand.__init__(self, options=options) |
| # This is sorta silly, but allows for unit testing: |
| self.expectations_factory = BotTestExpectationsFactory |
| |
| def _collect_expectation_lines(self, builder_names, factory): |
| all_lines = [] |
| for builder_name in builder_names: |
| model = TestExpectationsModel() |
| expectations = factory.expectations_for_builder(builder_name) |
| for line in expectations.expectation_lines(only_ignore_very_flaky=True): |
| model.add_expectation_line(line) |
| # FIXME: We need an official API to get all the test names or all test lines. |
| all_lines.extend(model._test_to_expectation_line.values()) |
| return all_lines |
| |
| def _commit_and_upload(self, tool, options): |
| files = tool.scm().changed_files() |
| flaky_tests_path = 'LayoutTests/FlakyTests' |
| if flaky_tests_path not in files: |
| print "%s is not changed, not uploading." % flaky_tests_path |
| return 0 |
| |
| if options.reviewers: |
| # FIXME: Could validate these as emails. sheriff_calendar has some code for that. |
| reviewer_emails = options.reviewers.split(',') |
| else: |
| reviewer_emails = sheriff_calendar.current_gardener_emails() |
| if not reviewer_emails: |
| print "No gardener, and --reviewers not specified, not bothering." |
| return 1 |
| |
| commit_message = self.COMMIT_MESSAGE % ','.join(reviewer_emails) |
| git_cmd = ['git', 'commit', '-m', commit_message, |
| tool.filesystem.join(tool.scm().checkout_root, flaky_tests_path)] |
| tool.executive.run_command(git_cmd) |
| |
| git_cmd = ['git', 'cl', 'upload', '--send-mail', '-f', |
| '--cc', ','.join(self.ALWAYS_CC)] |
| tool.executive.run_command(git_cmd) |
| |
| def execute(self, options, args, tool): |
| factory = self.expectations_factory() |
| |
| # FIXME: WebKit Linux 32 and WebKit Linux have the same specifiers; |
| # if we include both of them, we'll get duplicate lines. Ideally |
| # Linux 32 would have unique speicifiers. |
| most_builders = builders.all_builder_names() |
| if 'WebKit Linux 32' in most_builders: |
| most_builders.remove('WebKit Linux 32') |
| |
| lines = self._collect_expectation_lines(most_builders, factory) |
| lines.sort(key=lambda line: line.path) |
| |
| port = tool.port_factory.get() |
| # Skip any tests which are mentioned in the dashboard but not in our checkout: |
| fs = tool.filesystem |
| lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), line.path)), lines) |
| |
| # Note: This includes all flaky tests from the dashboard, even ones mentioned |
| # in existing TestExpectations. We could certainly load existing TestExpecations |
| # and filter accordingly, or update existing TestExpectations instead of FlakyTests. |
| flaky_tests_path = fs.join(port.layout_tests_dir(), 'FlakyTests') |
| flaky_tests_contents = self.FLAKY_TEST_CONTENTS % TestExpectations.list_to_string(lines) |
| fs.write_text_file(flaky_tests_path, flaky_tests_contents) |
| print "Updated %s" % flaky_tests_path |
| |
| if options.upload: |
| return self._commit_and_upload(tool, options) |
| |
