# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Updates TestExpectations based on results in builder bots.

Scans the TestExpectations file and uses results from actual builder bots runs
to remove tests that are marked as flaky but don't fail in the specified way.

E.g. If a test has this expectation:
    bug(test) fast/test.html [ Failure Pass ]

And all the runs on builders have passed the line will be removed.

Additionally, the runs don't all have to be Passing to remove the line;
as long as the non-Passing results are of a type not specified in the
expectation this line will be removed. For example, if this is the
expectation:

    bug(test) fast/test.html [ Crash Pass ]

But the results on the builders show only Passes and Timeouts, the line
will be removed since there's no Crash results.
"""

import argparse
import logging
import webbrowser

from webkitpy.layout_tests.models.test_expectations import TestExpectations
from webkitpy.tool.commands.flaky_tests import FlakyTests

_log = logging.getLogger(__name__)


def main(host, bot_test_expectations_factory, argv):
    parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('--verbose', '-v', action='store_true', default=False, help='enable more verbose logging')
    parser.add_argument('--show-results',
                        '-s',
                        action='store_true',
                        default=False,
                        help='Open results dashboard for all removed lines')
    args = parser.parse_args(argv)

    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format='%(levelname)s: %(message)s')

    port = host.port_factory.get()
    expectations_file = port.path_to_generic_test_expectations_file()
    if not host.filesystem.isfile(expectations_file):
        _log.warning("Didn't find generic expectations file at: " + expectations_file)
        return 1

    remove_flakes_o_matic = RemoveFlakesOMatic(host,
                                               port,
                                               bot_test_expectations_factory,
                                               webbrowser)

    test_expectations = remove_flakes_o_matic.get_updated_test_expectations()

    if args.show_results:
        remove_flakes_o_matic.show_removed_results()

    remove_flakes_o_matic.write_test_expectations(test_expectations,
                                                  expectations_file)
    return 0


class RemoveFlakesOMatic(object):

    def __init__(self, host, port, bot_test_expectations_factory, browser):
        self._host = host
        self._port = port
        self._expectations_factory = bot_test_expectations_factory
        self._builder_results_by_path = {}
        self._browser = browser
        self._expectations_to_remove_list = None

    def _can_delete_line(self, test_expectation_line):
        """Returns whether a given line in the expectations can be removed.

        Uses results from builder bots to determine if a given line is stale and
        can safely be removed from the TestExpectations file. (i.e. remove if
        the bots show that it's not flaky.) There are also some rules about when
        not to remove lines (e.g. never remove lines with Rebaseline
        expectations, don't remove non-flaky expectations, etc.)

        Args:
            test_expectation_line (TestExpectationLine): A line in the test
                expectation file to test for possible removal.

        Returns:
            True if the line can be removed, False otherwise.
        """
        expectations = test_expectation_line.expectations
        if len(expectations) < 2:
            return False

        # Don't check lines that have expectations like NeedsRebaseline or Skip.
        if self._has_unstrippable_expectations(expectations):
            return False

        # Don't check lines unless they're flaky. i.e. At least one expectation is a PASS.
        if not self._has_pass_expectation(expectations):
            return False

        # Don't check lines that have expectations for directories, since
        # the flakiness of all sub-tests isn't as easy to check.
        if self._port.test_isdir(test_expectation_line.name):
            return False

        # The line can be deleted if the only expectation on the line that appears in the actual
        # results is the PASS expectation.
        builders_checked = []
        for config in test_expectation_line.matching_configurations:
            builder_name = self._host.builders.builder_name_for_specifiers(config.version, config.build_type)

            if not builder_name:
                _log.debug('No builder with config %s', config)
                # For many configurations, there is no matching builder in
                # webkitpy/common/config/builders.py. We ignore these
                # configurations and make decisions based only on configurations
                # with actual builders.
                continue

            builders_checked.append(builder_name)

            if builder_name not in self._builder_results_by_path.keys():
                _log.error('Failed to find results for builder "%s"', builder_name)
                return False

            results_by_path = self._builder_results_by_path[builder_name]

            # No results means the tests were all skipped, or all results are passing.
            if test_expectation_line.path not in results_by_path.keys():
                continue

            results_for_single_test = results_by_path[test_expectation_line.path]

            if self._expectations_that_were_met(test_expectation_line, results_for_single_test) != set(['PASS']):
                return False

        if builders_checked:
            _log.debug('Checked builders:\n  %s', '\n  '.join(builders_checked))
        else:
            _log.warning('No matching builders for line, deleting line.')
        _log.info('Deleting line "%s"', test_expectation_line.original_string.strip())
        return True

    def _has_pass_expectation(self, expectations):
        return 'PASS' in expectations

    def _expectations_that_were_met(self, test_expectation_line, results_for_single_test):
        """Returns the set of expectations that appear in the given results.

        e.g. If the test expectations is "bug(test) fast/test.html [Crash Failure Pass]"
        and the results are ['TEXT', 'PASS', 'PASS', 'TIMEOUT'], then this method would
        return [Pass Failure] since the Failure expectation is satisfied by 'TEXT', Pass
        by 'PASS' but Crash doesn't appear in the results.

        Args:
            test_expectation_line: A TestExpectationLine object
            results_for_single_test: A list of result strings.
                e.g. ['IMAGE', 'IMAGE', 'PASS']

        Returns:
            A set containing expectations that occurred in the results.
        """
        # TODO(bokan): Does this not exist in a more central place?
        def replace_failing_with_fail(expectation):
            if expectation in ('TEXT', 'IMAGE', 'IMAGE+TEXT', 'AUDIO'):
                return 'FAIL'
            else:
                return expectation

        actual_results = {replace_failing_with_fail(r) for r in results_for_single_test}

        return set(test_expectation_line.expectations) & actual_results

    def _has_unstrippable_expectations(self, expectations):
        """Returns whether any of the given expectations are considered unstrippable.

        Unstrippable expectations are those which should stop a line from being
        removed regardless of builder bot results.

        Args:
            expectations: A list of string expectations.
                E.g. ['PASS', 'FAIL' 'CRASH']

        Returns:
            True if at least one of the expectations is unstrippable. False
            otherwise.
        """
        unstrippable_expectations = ('REBASELINE', 'NEEDSREBASELINE',
                                     'NEEDSMANUALREBASELINE', 'SLOW',
                                     'SKIP')
        return any(s in expectations for s in unstrippable_expectations)

    def _get_builder_results_by_path(self):
        """Returns a dictionary of results for each builder.

        Returns a dictionary where each key is a builder and value is a dictionary containing
        the distinct results for each test. E.g.

        {
            'WebKit Linux Precise': {
                  'test1.html': ['PASS', 'IMAGE'],
                  'test2.html': ['PASS'],
            },
            'WebKit Mac10.10': {
                  'test1.html': ['PASS', 'IMAGE'],
                  'test2.html': ['PASS', 'TEXT'],
            }
        }
        """
        builder_results_by_path = {}
        for builder_name in self._host.builders.all_continuous_builder_names():
            expectations_for_builder = (
                self._expectations_factory.expectations_for_builder(builder_name)
            )

            if not expectations_for_builder:
                # This is not fatal since we may not need to check these
                # results. If we do need these results we'll log an error later
                # when trying to check against them.
                _log.warning('Downloaded results are missing results for builder "%s"', builder_name)
                continue

            builder_results_by_path[builder_name] = (
                expectations_for_builder.all_results_by_path()
            )
        return builder_results_by_path

    def _remove_associated_comments_and_whitespace(self, expectations, removed_index):
        """Removes comments and whitespace from an empty expectation block.

        If the removed expectation was the last in a block of expectations, this method
        will remove any associated comments and whitespace.

        Args:
            expectations: A list of TestExpectationLine objects to be modified.
            removed_index: The index in the above list that was just removed.
        """
        was_last_expectation_in_block = (removed_index == len(expectations)
                                         or expectations[removed_index].is_whitespace()
                                         or expectations[removed_index].is_comment())

        # If the line immediately below isn't another expectation, then the block of
        # expectations definitely isn't empty so we shouldn't remove their associated comments.
        if not was_last_expectation_in_block:
            return

        did_remove_whitespace = False

        # We may have removed the last expectation in a block. Remove any whitespace above.
        while removed_index > 0 and expectations[removed_index - 1].is_whitespace():
            removed_index -= 1
            expectations.pop(removed_index)
            did_remove_whitespace = True

        # If we did remove some whitespace then we shouldn't remove any comments above it
        # since those won't have belonged to this expectation block. For example, commented
        # out expectations, or a section header.
        if did_remove_whitespace:
            return

        # Remove all comments above the removed line.
        while removed_index > 0 and expectations[removed_index - 1].is_comment():
            removed_index -= 1
            expectations.pop(removed_index)

        # Remove all whitespace above the comments.
        while removed_index > 0 and expectations[removed_index - 1].is_whitespace():
            removed_index -= 1
            expectations.pop(removed_index)

    def _expectations_to_remove(self):
        """Computes and returns the expectation lines that should be removed.

        Returns:
            A list of TestExpectationLine objects for lines that can be removed
            from the test expectations file. The result is memoized so that
            subsequent calls will not recompute the result.
        """
        if self._expectations_to_remove_list is not None:
            return self._expectations_to_remove_list

        self._builder_results_by_path = self._get_builder_results_by_path()
        self._expectations_to_remove_list = []
        test_expectations = TestExpectations(self._port, include_overrides=False).expectations()

        for expectation in test_expectations:
            if self._can_delete_line(expectation):
                self._expectations_to_remove_list.append(expectation)

        return self._expectations_to_remove_list

    def get_updated_test_expectations(self):
        """Filters out passing lines from TestExpectations file.

        Reads the current TestExpectations file and, using results from the
        build bots, removes lines that are passing. That is, removes lines that
        were not needed to keep the bots green.

        Returns:
            A TestExpectations object with the passing lines filtered out.
        """

        test_expectations = TestExpectations(self._port, include_overrides=False).expectations()
        for expectation in self._expectations_to_remove():
            index = test_expectations.index(expectation)
            test_expectations.remove(expectation)

            # Remove associated comments and whitespace if we've removed the last expectation under
            # a comment block. Only remove a comment block if it's not separated from the test
            # expectation line by whitespace.
            self._remove_associated_comments_and_whitespace(test_expectations, index)

        return test_expectations

    def show_removed_results(self):
        """Opens removed lines in the results dashboard.

        Opens the results dashboard in the browser, showing all the tests for lines that the script
        removed from the TestExpectations file and allowing the user to manually confirm the
        results.
        """
        removed_test_names = ','.join(x.name for x in self._expectations_to_remove())
        url = FlakyTests.FLAKINESS_DASHBOARD_URL % removed_test_names

        _log.info('Opening results dashboard: ' + url)
        self._browser.open(url)

    def write_test_expectations(self, test_expectations, test_expectations_file):
        """Writes the given TestExpectations object to the filesystem.

        Args:
            test_expectations: The TestExpectations object to write.
            test_expectations_file: The full file path of the Blink
                TestExpectations file. This file will be overwritten.
        """
        self._host.filesystem.write_text_file(
            test_expectations_file,
            TestExpectations.list_to_string(test_expectations, reconstitute_only_these=[]))