| # Copyright 2016 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Updates TestExpectations based on results in builder bots. |
| |
| Scans the TestExpectations file and uses results from actual builder bots runs |
| to remove tests that are marked as flaky but don't fail in the specified way. |
| |
| E.g. If a test has this expectation: |
| bug(test) fast/test.html [ Failure Pass ] |
| |
| And all the runs on builders have passed the line will be removed. |
| |
| Additionally, the runs don't all have to be Passing to remove the line; |
| as long as the non-Passing results are of a type not specified in the |
| expectation this line will be removed. For example, if this is the |
| expectation: |
| |
| bug(test) fast/test.html [ Crash Pass ] |
| |
| But the results on the builders show only Passes and Timeouts, the line |
| will be removed since there's no Crash results. |
| """ |
| |
| import argparse |
| import logging |
| import webbrowser |
| |
| from webkitpy.layout_tests.models.test_expectations import TestExpectations |
| from webkitpy.tool.commands.flaky_tests import FlakyTests |
| |
| _log = logging.getLogger(__name__) |
| |
| |
| def main(host, bot_test_expectations_factory, argv): |
| parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter) |
| parser.add_argument('--verbose', '-v', action='store_true', default=False, help='enable more verbose logging') |
| parser.add_argument('--show-results', |
| '-s', |
| action='store_true', |
| default=False, |
| help='Open results dashboard for all removed lines') |
| args = parser.parse_args(argv) |
| |
| logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format='%(levelname)s: %(message)s') |
| |
| port = host.port_factory.get() |
| expectations_file = port.path_to_generic_test_expectations_file() |
| if not host.filesystem.isfile(expectations_file): |
| _log.warning("Didn't find generic expectations file at: " + expectations_file) |
| return 1 |
| |
| remove_flakes_o_matic = RemoveFlakesOMatic(host, |
| port, |
| bot_test_expectations_factory, |
| webbrowser) |
| |
| test_expectations = remove_flakes_o_matic.get_updated_test_expectations() |
| |
| if args.show_results: |
| remove_flakes_o_matic.show_removed_results() |
| |
| remove_flakes_o_matic.write_test_expectations(test_expectations, |
| expectations_file) |
| return 0 |
| |
| |
| class RemoveFlakesOMatic(object): |
| |
| def __init__(self, host, port, bot_test_expectations_factory, browser): |
| self._host = host |
| self._port = port |
| self._expectations_factory = bot_test_expectations_factory |
| self._builder_results_by_path = {} |
| self._browser = browser |
| self._expectations_to_remove_list = None |
| |
| def _can_delete_line(self, test_expectation_line): |
| """Returns whether a given line in the expectations can be removed. |
| |
| Uses results from builder bots to determine if a given line is stale and |
| can safely be removed from the TestExpectations file. (i.e. remove if |
| the bots show that it's not flaky.) There are also some rules about when |
| not to remove lines (e.g. never remove lines with Rebaseline |
| expectations, don't remove non-flaky expectations, etc.) |
| |
| Args: |
| test_expectation_line (TestExpectationLine): A line in the test |
| expectation file to test for possible removal. |
| |
| Returns: |
| True if the line can be removed, False otherwise. |
| """ |
| expectations = test_expectation_line.expectations |
| if len(expectations) < 2: |
| return False |
| |
| # Don't check lines that have expectations like NeedsRebaseline or Skip. |
| if self._has_unstrippable_expectations(expectations): |
| return False |
| |
| # Don't check lines unless they're flaky. i.e. At least one expectation is a PASS. |
| if not self._has_pass_expectation(expectations): |
| return False |
| |
| # Don't check lines that have expectations for directories, since |
| # the flakiness of all sub-tests isn't as easy to check. |
| if self._port.test_isdir(test_expectation_line.name): |
| return False |
| |
| # The line can be deleted if the only expectation on the line that appears in the actual |
| # results is the PASS expectation. |
| builders_checked = [] |
| for config in test_expectation_line.matching_configurations: |
| builder_name = self._host.builders.builder_name_for_specifiers(config.version, config.build_type) |
| |
| if not builder_name: |
| _log.debug('No builder with config %s', config) |
| # For many configurations, there is no matching builder in |
| # webkitpy/common/config/builders.py. We ignore these |
| # configurations and make decisions based only on configurations |
| # with actual builders. |
| continue |
| |
| builders_checked.append(builder_name) |
| |
| if builder_name not in self._builder_results_by_path.keys(): |
| _log.error('Failed to find results for builder "%s"', builder_name) |
| return False |
| |
| results_by_path = self._builder_results_by_path[builder_name] |
| |
| # No results means the tests were all skipped, or all results are passing. |
| if test_expectation_line.path not in results_by_path.keys(): |
| continue |
| |
| results_for_single_test = results_by_path[test_expectation_line.path] |
| |
| if self._expectations_that_were_met(test_expectation_line, results_for_single_test) != set(['PASS']): |
| return False |
| |
| if builders_checked: |
| _log.debug('Checked builders:\n %s', '\n '.join(builders_checked)) |
| else: |
| _log.warning('No matching builders for line, deleting line.') |
| _log.info('Deleting line "%s"', test_expectation_line.original_string.strip()) |
| return True |
| |
| def _has_pass_expectation(self, expectations): |
| return 'PASS' in expectations |
| |
| def _expectations_that_were_met(self, test_expectation_line, results_for_single_test): |
| """Returns the set of expectations that appear in the given results. |
| |
| e.g. If the test expectations is "bug(test) fast/test.html [Crash Failure Pass]" |
| and the results are ['TEXT', 'PASS', 'PASS', 'TIMEOUT'], then this method would |
| return [Pass Failure] since the Failure expectation is satisfied by 'TEXT', Pass |
| by 'PASS' but Crash doesn't appear in the results. |
| |
| Args: |
| test_expectation_line: A TestExpectationLine object |
| results_for_single_test: A list of result strings. |
| e.g. ['IMAGE', 'IMAGE', 'PASS'] |
| |
| Returns: |
| A set containing expectations that occurred in the results. |
| """ |
| # TODO(bokan): Does this not exist in a more central place? |
| def replace_failing_with_fail(expectation): |
| if expectation in ('TEXT', 'IMAGE', 'IMAGE+TEXT', 'AUDIO'): |
| return 'FAIL' |
| else: |
| return expectation |
| |
| actual_results = {replace_failing_with_fail(r) for r in results_for_single_test} |
| |
| return set(test_expectation_line.expectations) & actual_results |
| |
| def _has_unstrippable_expectations(self, expectations): |
| """Returns whether any of the given expectations are considered unstrippable. |
| |
| Unstrippable expectations are those which should stop a line from being |
| removed regardless of builder bot results. |
| |
| Args: |
| expectations: A list of string expectations. |
| E.g. ['PASS', 'FAIL' 'CRASH'] |
| |
| Returns: |
| True if at least one of the expectations is unstrippable. False |
| otherwise. |
| """ |
| unstrippable_expectations = ('REBASELINE', 'NEEDSREBASELINE', |
| 'NEEDSMANUALREBASELINE', 'SLOW', |
| 'SKIP') |
| return any(s in expectations for s in unstrippable_expectations) |
| |
| def _get_builder_results_by_path(self): |
| """Returns a dictionary of results for each builder. |
| |
| Returns a dictionary where each key is a builder and value is a dictionary containing |
| the distinct results for each test. E.g. |
| |
| { |
| 'WebKit Linux Precise': { |
| 'test1.html': ['PASS', 'IMAGE'], |
| 'test2.html': ['PASS'], |
| }, |
| 'WebKit Mac10.10': { |
| 'test1.html': ['PASS', 'IMAGE'], |
| 'test2.html': ['PASS', 'TEXT'], |
| } |
| } |
| """ |
| builder_results_by_path = {} |
| for builder_name in self._host.builders.all_continuous_builder_names(): |
| expectations_for_builder = ( |
| self._expectations_factory.expectations_for_builder(builder_name) |
| ) |
| |
| if not expectations_for_builder: |
| # This is not fatal since we may not need to check these |
| # results. If we do need these results we'll log an error later |
| # when trying to check against them. |
| _log.warning('Downloaded results are missing results for builder "%s"', builder_name) |
| continue |
| |
| builder_results_by_path[builder_name] = ( |
| expectations_for_builder.all_results_by_path() |
| ) |
| return builder_results_by_path |
| |
| def _remove_associated_comments_and_whitespace(self, expectations, removed_index): |
| """Removes comments and whitespace from an empty expectation block. |
| |
| If the removed expectation was the last in a block of expectations, this method |
| will remove any associated comments and whitespace. |
| |
| Args: |
| expectations: A list of TestExpectationLine objects to be modified. |
| removed_index: The index in the above list that was just removed. |
| """ |
| was_last_expectation_in_block = (removed_index == len(expectations) |
| or expectations[removed_index].is_whitespace() |
| or expectations[removed_index].is_comment()) |
| |
| # If the line immediately below isn't another expectation, then the block of |
| # expectations definitely isn't empty so we shouldn't remove their associated comments. |
| if not was_last_expectation_in_block: |
| return |
| |
| did_remove_whitespace = False |
| |
| # We may have removed the last expectation in a block. Remove any whitespace above. |
| while removed_index > 0 and expectations[removed_index - 1].is_whitespace(): |
| removed_index -= 1 |
| expectations.pop(removed_index) |
| did_remove_whitespace = True |
| |
| # If we did remove some whitespace then we shouldn't remove any comments above it |
| # since those won't have belonged to this expectation block. For example, commented |
| # out expectations, or a section header. |
| if did_remove_whitespace: |
| return |
| |
| # Remove all comments above the removed line. |
| while removed_index > 0 and expectations[removed_index - 1].is_comment(): |
| removed_index -= 1 |
| expectations.pop(removed_index) |
| |
| # Remove all whitespace above the comments. |
| while removed_index > 0 and expectations[removed_index - 1].is_whitespace(): |
| removed_index -= 1 |
| expectations.pop(removed_index) |
| |
| def _expectations_to_remove(self): |
| """Computes and returns the expectation lines that should be removed. |
| |
| Returns: |
| A list of TestExpectationLine objects for lines that can be removed |
| from the test expectations file. The result is memoized so that |
| subsequent calls will not recompute the result. |
| """ |
| if self._expectations_to_remove_list is not None: |
| return self._expectations_to_remove_list |
| |
| self._builder_results_by_path = self._get_builder_results_by_path() |
| self._expectations_to_remove_list = [] |
| test_expectations = TestExpectations(self._port, include_overrides=False).expectations() |
| |
| for expectation in test_expectations: |
| if self._can_delete_line(expectation): |
| self._expectations_to_remove_list.append(expectation) |
| |
| return self._expectations_to_remove_list |
| |
| def get_updated_test_expectations(self): |
| """Filters out passing lines from TestExpectations file. |
| |
| Reads the current TestExpectations file and, using results from the |
| build bots, removes lines that are passing. That is, removes lines that |
| were not needed to keep the bots green. |
| |
| Returns: |
| A TestExpectations object with the passing lines filtered out. |
| """ |
| |
| test_expectations = TestExpectations(self._port, include_overrides=False).expectations() |
| for expectation in self._expectations_to_remove(): |
| index = test_expectations.index(expectation) |
| test_expectations.remove(expectation) |
| |
| # Remove associated comments and whitespace if we've removed the last expectation under |
| # a comment block. Only remove a comment block if it's not separated from the test |
| # expectation line by whitespace. |
| self._remove_associated_comments_and_whitespace(test_expectations, index) |
| |
| return test_expectations |
| |
| def show_removed_results(self): |
| """Opens removed lines in the results dashboard. |
| |
| Opens the results dashboard in the browser, showing all the tests for lines that the script |
| removed from the TestExpectations file and allowing the user to manually confirm the |
| results. |
| """ |
| removed_test_names = ','.join(x.name for x in self._expectations_to_remove()) |
| url = FlakyTests.FLAKINESS_DASHBOARD_URL % removed_test_names |
| |
| _log.info('Opening results dashboard: ' + url) |
| self._browser.open(url) |
| |
| def write_test_expectations(self, test_expectations, test_expectations_file): |
| """Writes the given TestExpectations object to the filesystem. |
| |
| Args: |
| test_expectations: The TestExpectations object to write. |
| test_expectations_file: The full file path of the Blink |
| TestExpectations file. This file will be overwritten. |
| """ |
| self._host.filesystem.write_text_file( |
| test_expectations_file, |
| TestExpectations.list_to_string(test_expectations, reconstitute_only_these=[])) |
