# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import unittest

from webkitpy.common.host_mock import MockHost
from webkitpy.layout_tests.models import test_expectations
from webkitpy.layout_tests.models import test_failures
from webkitpy.layout_tests.models import test_results
from webkitpy.layout_tests.models import test_run_results


def get_result(test_name, result_type=test_expectations.PASS, run_time=0):
    failures = []
    if result_type == test_expectations.TIMEOUT:
        failures = [test_failures.FailureTimeout()]
    elif result_type == test_expectations.AUDIO:
        failures = [test_failures.FailureAudioMismatch()]
    elif result_type == test_expectations.TEXT:
        failures = [test_failures.FailureTextMismatch()]
    elif result_type == test_expectations.IMAGE:
        failures = [test_failures.FailureImageHashMismatch()]
    elif result_type == test_expectations.CRASH:
        failures = [test_failures.FailureCrash()]
    elif result_type == test_expectations.LEAK:
        failures = [test_failures.FailureLeak()]
    return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)


def run_results(port, extra_skipped_tests=None):
    tests = [
        'passes/text.html',
        'failures/expected/timeout.html',
        'failures/expected/crash.html',
        'failures/expected/leak.html',
        'failures/expected/keyboard.html',
        'failures/expected/audio.html',
        'failures/expected/text.html',
        'passes/skipped/skip.html'
    ]
    expectations = test_expectations.TestExpectations(port, tests)
    if extra_skipped_tests:
        expectations.add_extra_skipped_tests(extra_skipped_tests)
    return test_run_results.TestRunResults(expectations, len(tests))


def summarized_results(port, expected, passing, flaky, only_include_failing=False, extra_skipped_tests=None):
    test_is_slow = False

    all_retry_results = []
    initial_results = run_results(port, extra_skipped_tests)
    if expected:
        initial_results.add(get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/leak.html', test_expectations.LEAK), expected, test_is_slow)
    elif passing:
        skipped_result = get_result('passes/skipped/skip.html')
        skipped_result.type = test_expectations.SKIP
        initial_results.add(skipped_result, expected, test_is_slow)

        initial_results.add(get_result('passes/text.html', run_time=1), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/audio.html'), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/timeout.html'), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/crash.html'), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow)
    else:
        initial_results.add(get_result('passes/text.html', test_expectations.TIMEOUT, run_time=1), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/audio.html',
                                       test_expectations.CRASH, run_time=0.049), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/timeout.html',
                                       test_expectations.TEXT, run_time=0.05), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
        initial_results.add(get_result('failures/expected/leak.html', test_expectations.TIMEOUT), expected, test_is_slow)

        # we only list keyboard.html here, since normally this is WontFix
        initial_results.add(get_result('failures/expected/keyboard.html', test_expectations.SKIP), expected, test_is_slow)

        initial_results.add(get_result('failures/expected/text.html', test_expectations.IMAGE), expected, test_is_slow)

        all_retry_results = [run_results(port, extra_skipped_tests),
                             run_results(port, extra_skipped_tests),
                             run_results(port, extra_skipped_tests)]

        def add_result_to_all_retries(new_result, expected):
            for run_result in all_retry_results:
                run_result.add(new_result, expected, test_is_slow)

        if flaky:
            add_result_to_all_retries(get_result('passes/text.html', test_expectations.PASS), True)
            add_result_to_all_retries(
                get_result('failures/expected/audio.html', test_expectations.AUDIO), True)
            add_result_to_all_retries(
                get_result('failures/expected/leak.html', test_expectations.LEAK), True)
            add_result_to_all_retries(
                get_result('failures/expected/timeout.html', test_expectations.AUDIO), True)

            all_retry_results[0].add(
                get_result('failures/expected/crash.html', test_expectations.AUDIO),
                False, test_is_slow)
            all_retry_results[1].add(
                get_result('failures/expected/crash.html', test_expectations.CRASH),
                True, test_is_slow)
            all_retry_results[2].add(
                get_result('failures/expected/crash.html', test_expectations.LEAK),
                False, test_is_slow)

            all_retry_results[0].add(
                get_result('failures/expected/text.html', test_expectations.TEXT),
                True, test_is_slow)

        else:
            add_result_to_all_retries(
                get_result('passes/text.html', test_expectations.TIMEOUT), False)
            add_result_to_all_retries(
                get_result('failures/expected/audio.html', test_expectations.LEAK), False)
            add_result_to_all_retries(
                get_result('failures/expected/crash.html', test_expectations.TIMEOUT), False)
            add_result_to_all_retries(
                get_result('failures/expected/leak.html', test_expectations.TIMEOUT), False)

            all_retry_results[0].add(
                get_result('failures/expected/timeout.html', test_expectations.AUDIO),
                False, test_is_slow)
            all_retry_results[1].add(
                get_result('failures/expected/timeout.html', test_expectations.CRASH),
                False, test_is_slow)
            all_retry_results[2].add(
                get_result('failures/expected/timeout.html', test_expectations.LEAK),
                False, test_is_slow)

    return test_run_results.summarize_results(
        port, initial_results.expectations, initial_results, all_retry_results,
        enabled_pixel_tests_in_retry=False,
        only_include_failing=only_include_failing)


class InterpretTestFailuresTest(unittest.TestCase):

    def setUp(self):
        host = MockHost()
        self.port = host.port_factory.get(port_name='test')

    def test_interpret_test_failures(self):
        test_dict = test_run_results._interpret_test_failures(
            [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
        self.assertEqual(len(test_dict), 0)

        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()])
        self.assertIn('is_missing_audio', test_dict)

        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingResult()])
        self.assertIn('is_missing_text', test_dict)

        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImage()])
        self.assertIn('is_missing_image', test_dict)

        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImageHash()])
        self.assertIn('is_missing_image', test_dict)


class SummarizedResultsTest(unittest.TestCase):

    def setUp(self):
        host = MockHost()
        self.port = host.port_factory.get(port_name='test')

    def test_no_chromium_revision(self):
        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
        self.assertNotIn('revision', summary)

    def test_num_failures_by_type(self):
        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
        self.assertEquals(
            summary['num_failures_by_type'],
            {
                'CRASH': 1,
                'MISSING': 0,
                'TEXT': 1,
                'IMAGE': 1,
                'NEEDSREBASELINE': 0,
                'NEEDSMANUALREBASELINE': 0,
                'PASS': 0,
                'REBASELINE': 0,
                'SKIP': 0,
                'SLOW': 0,
                'TIMEOUT': 3,
                'IMAGE+TEXT': 0,
                'LEAK': 0,
                'FAIL': 0,
                'AUDIO': 0,
                'WONTFIX': 1
            })

        summary = summarized_results(self.port, expected=True, passing=False, flaky=False)
        self.assertEquals(
            summary['num_failures_by_type'],
            {
                'CRASH': 1,
                'MISSING': 0,
                'TEXT': 0,
                'IMAGE': 0,
                'NEEDSREBASELINE': 0,
                'NEEDSMANUALREBASELINE': 0,
                'PASS': 1,
                'REBASELINE': 0,
                'SKIP': 0,
                'SLOW': 0,
                'TIMEOUT': 1,
                'IMAGE+TEXT': 0,
                'LEAK': 1,
                'FAIL': 0,
                'AUDIO': 1,
                'WONTFIX': 0
            })

        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
        self.assertEquals(
            summary['num_failures_by_type'],
            {
                'CRASH': 0,
                'MISSING': 0,
                'TEXT': 0,
                'IMAGE': 0,
                'NEEDSREBASELINE': 0,
                'NEEDSMANUALREBASELINE': 0,
                'PASS': 5,
                'REBASELINE': 0,
                'SKIP': 1,
                'SLOW': 0,
                'TIMEOUT': 0,
                'IMAGE+TEXT': 0,
                'LEAK': 0,
                'FAIL': 0,
                'AUDIO': 0,
                'WONTFIX': 0
            })

    def test_chromium_revision(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
        self.assertNotEquals(summary['chromium_revision'],
                             '')

    def test_bug_entry(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
        self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bugs'], ['Bug(test)'])

    def test_extra_skipped_tests(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=False, passing=True, flaky=False, extra_skipped_tests=['passes/text.html'])
        self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'NOTRUN')

    def test_summarized_results_wontfix(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
        self.assertEquals(summary['tests']['failures']['expected']['keyboard.html']['expected'], 'WONTFIX CRASH')
        self.assertTrue(summary['tests']['passes']['text.html']['is_unexpected'])
        self.assertEqual(summary['num_passes'], 1)
        self.assertEqual(summary['num_regressions'], 6)
        self.assertEqual(summary['num_flaky'], 0)

    def test_summarized_results_expected_pass(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
        self.assertTrue(summary['tests']['passes']['text.html'])
        self.assertTrue('is_unexpected' not in summary['tests']['passes']['text.html'])
        self.assertEqual(summary['num_passes'], 5)
        self.assertEqual(summary['num_regressions'], 0)
        self.assertEqual(summary['num_flaky'], 0)

    def test_summarized_results_expected_only_include_failing(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=True, passing=False, flaky=False, only_include_failing=True)
        self.assertNotIn('passes', summary['tests'])
        self.assertTrue(summary['tests']['failures']['expected']['audio.html'])
        self.assertTrue(summary['tests']['failures']['expected']['timeout.html'])
        self.assertTrue(summary['tests']['failures']['expected']['crash.html'])
        self.assertTrue(summary['tests']['failures']['expected']['leak.html'])
        self.assertEqual(summary['num_passes'], 1)
        self.assertEqual(summary['num_regressions'], 0)
        self.assertEqual(summary['num_flaky'], 0)

    def test_summarized_results_skipped(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
        self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['expected'], 'SKIP')

    def test_summarized_results_only_include_failing(self):
        self.port._options.builder_name = 'dummy builder'
        summary = summarized_results(self.port, expected=False, passing=True, flaky=False, only_include_failing=True)
        self.assertTrue('passes' not in summary['tests'])
        self.assertEqual(summary['num_passes'], 5)
        self.assertEqual(summary['num_regressions'], 0)
        self.assertEqual(summary['num_flaky'], 0)

    def test_rounded_run_times(self):
        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
        self.assertEquals(summary['tests']['passes']['text.html']['time'], 1)
        self.assertTrue('time' not in summary['tests']['failures']['expected']['audio.html'])
        self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['time'], 0.1)
        self.assertTrue('time' not in summary['tests']['failures']['expected']['crash.html'])
        self.assertTrue('time' not in summary['tests']['failures']['expected']['leak.html'])

    def test_timeout_then_unexpected_pass(self):
        test_name = 'failures/expected/text.html'
        expectations = test_expectations.TestExpectations(self.port, [test_name])
        initial_results = test_run_results.TestRunResults(expectations, 1)
        initial_results.add(get_result(test_name, test_expectations.TIMEOUT, run_time=1), False, False)
        all_retry_results = [test_run_results.TestRunResults(expectations, 1),
                             test_run_results.TestRunResults(expectations, 1),
                             test_run_results.TestRunResults(expectations, 1)]
        all_retry_results[0].add(get_result(test_name, test_expectations.LEAK, run_time=0.1), False, False)
        all_retry_results[1].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
        all_retry_results[2].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
        summary = test_run_results.summarize_results(
            self.port, expectations, initial_results, all_retry_results,
            enabled_pixel_tests_in_retry=True)
        self.assertIn('is_unexpected', summary['tests']['failures']['expected']['text.html'])
        self.assertEquals(summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL')
        self.assertEquals(summary['tests']['failures']['expected']['text.html']['actual'], 'TIMEOUT LEAK PASS PASS')
        self.assertEquals(summary['num_passes'], 1)
        self.assertEquals(summary['num_regressions'], 0)
        self.assertEquals(summary['num_flaky'], 0)

    def test_summarized_results_flaky(self):
        summary = summarized_results(self.port, expected=False, passing=False, flaky=True)

        self.assertTrue('is_unexpected' not in summary['tests']['failures']['expected']['crash.html'])
        self.assertEquals(summary['tests']['failures']['expected']['crash.html']['expected'], 'CRASH')
        self.assertEquals(summary['tests']['failures']['expected']['crash.html']['actual'], 'TIMEOUT AUDIO CRASH LEAK')

        self.assertTrue('is_unexpected' not in summary['tests']['passes']['text.html'])
        self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS')
        self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'TIMEOUT PASS PASS PASS')

        self.assertTrue(summary['tests']['failures']['expected']['timeout.html']['is_unexpected'])
        self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['expected'], 'TIMEOUT')
        self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['actual'], 'TEXT AUDIO AUDIO AUDIO')

        self.assertTrue('is_unexpected' not in summary['tests']['failures']['expected']['leak.html'])
        self.assertEquals(summary['tests']['failures']['expected']['leak.html']['expected'], 'LEAK')
        self.assertEquals(summary['tests']['failures']['expected']['leak.html']['actual'], 'TIMEOUT LEAK LEAK LEAK')

        self.assertTrue('is_unexpected' not in summary['tests']['failures']['expected']['audio.html'])
        self.assertEquals(summary['tests']['failures']['expected']['audio.html']['expected'], 'FAIL')
        self.assertEquals(summary['tests']['failures']['expected']['audio.html']['actual'], 'CRASH AUDIO AUDIO AUDIO')

        self.assertEquals(summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL')
        self.assertTrue('is_unexpected' not in summary['tests']['failures']['expected']['text.html'])

        self.assertEquals(summary['num_flaky'], 6)
        self.assertEquals(summary['num_passes'], 1)  # keyboard.html
        self.assertEquals(summary['num_regressions'], 0)

    def test_summarized_results_flaky_pass_after_first_retry(self):
        test_name = 'passes/text.html'
        expectations = test_expectations.TestExpectations(self.port, [test_name])
        initial_results = test_run_results.TestRunResults(expectations, 1)
        initial_results.add(get_result(test_name, test_expectations.CRASH), False, False)
        all_retry_results = [test_run_results.TestRunResults(expectations, 1),
                             test_run_results.TestRunResults(expectations, 1),
                             test_run_results.TestRunResults(expectations, 1)]
        all_retry_results[0].add(get_result(test_name, test_expectations.TIMEOUT), False, False)
        all_retry_results[1].add(get_result(test_name, test_expectations.PASS), True, False)
        all_retry_results[2].add(get_result(test_name, test_expectations.PASS), True, False)
        summary = test_run_results.summarize_results(
            self.port, expectations, initial_results, all_retry_results,
            enabled_pixel_tests_in_retry=True)
        self.assertTrue('is_unexpected' not in summary['tests']['passes']['text.html'])
        self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS')
        self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'CRASH TIMEOUT PASS PASS')
        self.assertEquals(summary['num_flaky'], 1)
        self.assertEquals(summary['num_passes'], 0)
        self.assertEquals(summary['num_regressions'], 0)

    def test_summarized_results_regression(self):
        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)

        self.assertTrue(summary['tests']['failures']['expected']['timeout.html']['is_unexpected'])
        self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['expected'], 'TIMEOUT')
        self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['actual'], 'TEXT AUDIO CRASH LEAK')

        self.assertTrue(summary['tests']['passes']['text.html']['is_unexpected'])
        self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS')
        self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'TIMEOUT')

        self.assertTrue(summary['tests']['failures']['expected']['crash.html']['is_unexpected'])
        self.assertEquals(summary['tests']['failures']['expected']['crash.html']['expected'], 'CRASH')
        self.assertEquals(summary['tests']['failures']['expected']['crash.html']['actual'], 'TIMEOUT')

        self.assertTrue(summary['tests']['failures']['expected']['leak.html']['is_unexpected'])
        self.assertEquals(summary['tests']['failures']['expected']['leak.html']['expected'], 'LEAK')
        self.assertEquals(summary['tests']['failures']['expected']['leak.html']['actual'], 'TIMEOUT')

        self.assertTrue(summary['tests']['failures']['expected']['audio.html']['is_unexpected'])
        self.assertEquals(summary['tests']['failures']['expected']['audio.html']['expected'], 'FAIL')
        self.assertEquals(summary['tests']['failures']['expected']['audio.html']['actual'], 'CRASH LEAK LEAK LEAK')

        self.assertEquals(summary['num_regressions'], 6)
        self.assertEquals(summary['num_passes'], 1)  # keyboard.html
        self.assertEquals(summary['num_flaky'], 0)

    def test_results_contains_path_delimiter(self):
        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
        self.assertEqual(summary['path_delimiter'], '/')