# Copyright (C) 2013 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the Google name nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Generates a fake TestExpectations file consisting of flaky tests from the bot
corresponding to the give port.
"""

import json
import logging
import os.path
import urllib
import urllib2

from webkitpy.layout_tests.models.test_expectations import TestExpectations, PASS
from webkitpy.layout_tests.models.test_expectations import TestExpectationLine


_log = logging.getLogger(__name__)


class ResultsJSON(object):
    """Contains the contents of a results.json file.

    results.json v4 format:
    {
      'version': 4,
      'builder name' : {
        'blinkRevision': [],
        'tests': {
          'directory' { # Each path component is a dictionary.
            'testname.html': {
              'expected' : 'FAIL', # Expectation name.
              'results': [], # Run-length encoded result.
              'times': [],
              'bugs': [], # Bug URLs.
            }
          }
        }
      }
      'buildNumbers': [],
      'secondsSinceEpoch': [],
      'chromeRevision': [],
      'failure_map': {}  # Map from letter code to expectation name.
    }
    """
    TESTS_KEY = 'tests'
    FAILURE_MAP_KEY = 'failure_map'
    RESULTS_KEY = 'results'
    EXPECTATIONS_KEY = 'expected'
    BUGS_KEY = 'bugs'
    RLE_LENGTH = 0
    RLE_VALUE = 1

    # results.json was originally designed to support
    # multiple builders in one json file, so the builder_name
    # is needed to figure out which builder this json file
    # refers to (and thus where the results are stored)
    def __init__(self, builder_name, json_dict):
        self.builder_name = builder_name
        self._json = json_dict

    def _walk_trie(self, trie, parent_path):
        for name, value in trie.items():
            full_path = os.path.join(parent_path, name)

            # FIXME: If we ever have a test directory self.RESULTS_KEY
            # ("results"), this logic will break!
            if self.RESULTS_KEY not in value:
                for path, results in self._walk_trie(value, full_path):
                    yield path, results
            else:
                yield full_path, value

    def walk_results(self, full_path=''):
        tests_trie = self._json[self.builder_name][self.TESTS_KEY]
        return self._walk_trie(tests_trie, parent_path='')

    def expectation_for_type(self, type_char):
        return self._json[self.builder_name][self.FAILURE_MAP_KEY][type_char]

    # Knowing how to parse the run-length-encoded values in results.json
    # is a detail of this class.
    def occurances_and_type_from_result_item(self, item):
        return item[self.RLE_LENGTH], item[self.RLE_VALUE]


class BotTestExpectationsFactory(object):
    RESULTS_URL_PREFIX = (
        'https://test-results.appspot.com/testfile?master=chromium.webkit&'
        'testtype=webkit_tests&name=results-small.json&builder=')

    def __init__(self, builders):
        self.builders = builders

    def _results_json_for_port(self, port_name, builder_category):
        builder = self.builders.builder_name_for_port_name(port_name)
        if not builder:
            return None
        return self._results_json_for_builder(builder)

    def _results_json_for_builder(self, builder):
        results_url = self.RESULTS_URL_PREFIX + urllib.quote(builder)
        try:
            _log.debug('Fetching flakiness data from appengine.')
            return ResultsJSON(builder, json.load(urllib2.urlopen(results_url)))
        except urllib2.URLError as error:
            _log.warning('Could not retrieve flakiness data from the bot.  url: %s', results_url)
            _log.warning(error)

    def expectations_for_port(self, port_name, builder_category='layout'):
        # FIXME: This only grabs release builder's flakiness data. If we're running debug,
        # when we should grab the debug builder's data.
        # FIXME: What should this do if there is no debug builder for a port, e.g. we have
        # no debug XP builder? Should it use the release bot or another Windows debug bot?
        # At the very least, it should log an error.
        results_json = self._results_json_for_port(port_name, builder_category)
        if not results_json:
            return None
        return BotTestExpectations(results_json, self.builders)

    def expectations_for_builder(self, builder):
        results_json = self._results_json_for_builder(builder)
        if not results_json:
            return None
        return BotTestExpectations(results_json, self.builders)


class BotTestExpectations(object):
    # FIXME: Get this from the json instead of hard-coding it.
    RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y']  # NO_DATA, SKIP, NOTRUN

    # TODO(ojan): Remove this once crbug.com/514378 is fixed.
    # The JSON can contain results for expectations, not just actual result types.
    NON_RESULT_TYPES = ['S', 'X']  # SLOW, SKIP

    # specifiers arg is used in unittests to avoid the static dependency on builders.
    def __init__(self, results_json, builders, specifiers=None):
        self.results_json = results_json
        self.specifiers = specifiers or set(builders.specifiers_for_builder(results_json.builder_name))

    def _line_from_test_and_flaky_types(self, test_path, flaky_types):
        line = TestExpectationLine()
        line.original_string = test_path
        line.name = test_path
        line.filename = test_path
        line.path = test_path  # FIXME: Should this be normpath?
        line.matching_tests = [test_path]
        line.bugs = ['crbug.com/FILE_A_BUG_BEFORE_COMMITTING_THIS']
        line.expectations = sorted(flaky_types)
        line.specifiers = self.specifiers
        return line

    def flakes_by_path(self, only_ignore_very_flaky):
        """Sets test expectations to bot results if there are at least two distinct results."""
        flakes_by_path = {}
        for test_path, entry in self.results_json.walk_results():
            flaky_types = self._flaky_types_in_results(entry, only_ignore_very_flaky)
            if len(flaky_types) <= 1:
                continue
            flakes_by_path[test_path] = sorted(flaky_types)
        return flakes_by_path

    def unexpected_results_by_path(self):
        """For tests with unexpected results, returns original expectations + results."""
        def exp_to_string(exp):
            return TestExpectations.EXPECTATIONS_TO_STRING.get(exp, None).upper()

        def string_to_exp(string):
            # Needs a bit more logic than the method above,
            # since a PASS is 0 and evaluates to False.
            result = TestExpectations.EXPECTATIONS.get(string.lower(), None)
            if not result is None:
                return result
            raise ValueError(string)

        unexpected_results_by_path = {}
        for test_path, entry in self.results_json.walk_results():
            # Expectations for this test. No expectation defaults to PASS.
            exp_string = entry.get(self.results_json.EXPECTATIONS_KEY, u'PASS')

            # All run-length-encoded results for this test.
            results_dict = entry.get(self.results_json.RESULTS_KEY, {})

            # Set of expectations for this test.
            expectations = set(map(string_to_exp, exp_string.split(' ')))

            # Set of distinct results for this test.
            result_types = self._all_types_in_results(results_dict)

            # Distinct results as non-encoded strings.
            result_strings = map(self.results_json.expectation_for_type, result_types)

            # Distinct resulting expectations.
            result_exp = map(string_to_exp, result_strings)

            expected = lambda e: TestExpectations.result_was_expected(e, expectations, False)

            additional_expectations = set(e for e in result_exp if not expected(e))

            # Test did not have unexpected results.
            if not additional_expectations:
                continue

            expectations.update(additional_expectations)
            unexpected_results_by_path[test_path] = sorted(map(exp_to_string, expectations))
        return unexpected_results_by_path

    def all_results_by_path(self):
        """Returns all seen result types for each test.

        Returns a dictionary from each test path that has a result to a list of distinct, sorted result
        strings. For example, if the test results are as follows:

            a.html IMAGE IMAGE PASS PASS PASS TIMEOUT PASS TEXT
            b.html PASS PASS PASS PASS PASS PASS PASS PASS
            c.html

        This method will return:
            {
                'a.html': ['IMAGE', 'TEXT', 'TIMEOUT', 'PASS'],
                'b.html': ['PASS'],
            }
        """
        results_by_path = {}
        for test_path, entry in self.results_json.walk_results():
            results_dict = entry.get(self.results_json.RESULTS_KEY, {})

            result_types = self._all_types_in_results(results_dict)

            if not result_types:
                continue

            # Distinct results as non-encoded strings.
            result_strings = map(self.results_json.expectation_for_type, result_types)

            results_by_path[test_path] = sorted(result_strings)
        return results_by_path

    def expectation_lines(self, only_ignore_very_flaky):
        lines = []
        for test_path, entry in self.results_json.walk_results():
            flaky_types = self._flaky_types_in_results(entry, only_ignore_very_flaky)
            if len(flaky_types) > 1:
                line = self._line_from_test_and_flaky_types(test_path, flaky_types)
                lines.append(line)
        return lines

    def _all_types_in_results(self, run_length_encoded_results):
        results = set()

        for result_item in run_length_encoded_results:
            _, result_types = self.results_json.occurances_and_type_from_result_item(result_item)

            for result_type in result_types:
                if result_type not in self.RESULT_TYPES_TO_IGNORE:
                    results.add(result_type)

        return results

    def _result_to_enum(self, result):
        return TestExpectations.EXPECTATIONS[result.lower()]

    def _flaky_types_in_results(self, results_entry, only_ignore_very_flaky):
        flaky_results = set()

        # Always include pass as an expected result. Passes will never turn the bot red.
        # This fixes cases where the expectations have an implicit Pass, e.g. [ Slow ].
        latest_expectations = [PASS]
        if self.results_json.EXPECTATIONS_KEY in results_entry:
            expectations_list = results_entry[self.results_json.EXPECTATIONS_KEY].split(' ')
            latest_expectations += [self._result_to_enum(expectation) for expectation in expectations_list]

        for result_item in results_entry[self.results_json.RESULTS_KEY]:
            _, result_types_str = self.results_json.occurances_and_type_from_result_item(result_item)

            result_types = []
            for result_type in result_types_str:
                # TODO(ojan): Remove this if-statement once crbug.com/514378 is fixed.
                if result_type not in self.NON_RESULT_TYPES:
                    result_types.append(self.results_json.expectation_for_type(result_type))

            # It didn't flake if it didn't retry.
            if len(result_types) <= 1:
                continue

            # If the test ran as expected after only one retry, it's not very flaky.
            # It's only very flaky if it failed the first run and the first retry
            # and then ran as expected in one of the subsequent retries.
            # If there are only two entries, then that means it failed on the first
            # try and ran as expected on the second because otherwise we'd have
            # a third entry from the next try.
            if only_ignore_very_flaky and len(result_types) == 2:
                continue

            has_unexpected_results = False
            for result_type in result_types:
                result_enum = self._result_to_enum(result_type)
                # TODO(ojan): We really should be grabbing the expected results from the time
                # of the run instead of looking at the latest expected results. That's a lot
                # more complicated though. So far we've been looking at the aggregated
                # results_small.json off test_results.appspot, which has all the information
                # for the last 100 runs. In order to do this, we'd need to look at the
                # individual runs' full_results.json, which would be slow and more complicated.
                # The only thing we lose by not fixing this is that a test that was flaky
                # and got fixed will still get printed out until 100 runs have passed.
                if not TestExpectations.result_was_expected(result_enum, latest_expectations, test_needs_rebaselining=False):
                    has_unexpected_results = True
                    break

            if has_unexpected_results:
                flaky_results = flaky_results.union(set(result_types))

        return flaky_results