# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import optparse
import unittest

from webkitpy.common.net.buildbot import Build
from webkitpy.common.net.layout_test_results import LayoutTestResults
from webkitpy.common.system.executive_mock import MockExecutive
from webkitpy.common.system.output_capture import OutputCapture
from webkitpy.layout_tests.builder_list import BuilderList
from webkitpy.tool.commands.rebaseline import (
    AbstractParallelRebaselineCommand, CopyExistingBaselinesInternal,
    Rebaseline, RebaselineExpectations, RebaselineJson, RebaselineTest
)
from webkitpy.tool.mock_tool import MockWebKitPatch


# pylint: disable=protected-access
class BaseTestCase(unittest.TestCase):
    WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/MOCK_Mac10_11/results/layout-test-results'

    command_constructor = None

    def setUp(self):
        self.tool = MockWebKitPatch()
        # Lint warns that command_constructor might not be set, but this is intentional; pylint: disable=not-callable
        self.command = self.command_constructor()
        self.command._tool = self.tool
        self.tool.builders = BuilderList({
            'MOCK Mac10.10 (dbg)': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Debug']},
            'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']},
            'MOCK Mac10.11 (dbg)': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Debug']},
            'MOCK Mac10.11 ASAN': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
            'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
            'MOCK Precise': {'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release']},
            'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']},
            'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']},
            'MOCK Win7 (dbg)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
            'MOCK Win7 (dbg)(1)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
            'MOCK Win7 (dbg)(2)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
            'MOCK Win7': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release']},
        })
        self.mac_port = self.tool.port_factory.get_from_builder_name('MOCK Mac10.11')

        self.mac_expectations_path = self.mac_port.path_to_generic_test_expectations_file()
        self.tool.filesystem.write_text_file(
            self.tool.filesystem.join(self.mac_port.layout_tests_dir(), 'VirtualTestSuites'), '[]')

        # In AbstractParallelRebaselineCommand._rebaseline_commands, a default port
        # object is gotten using self.tool.port_factory.get(), which is used to get
        # test paths -- and the layout tests directory may be different for the "test"
        # ports and real ports. Since only "test" ports are used in this class,
        # we can make the default port also a "test" port.
        self.original_port_factory_get = self.tool.port_factory.get
        test_port = self.tool.port_factory.get('test')

        def get_test_port(port_name=None, options=None, **kwargs):
            if not port_name:
                return test_port
            return self.original_port_factory_get(port_name, options, **kwargs)

        self.tool.port_factory.get = get_test_port

    def tearDown(self):
        self.tool.port_factory.get = self.original_port_factory_get

    def _expand(self, path):
        if self.tool.filesystem.isabs(path):
            return path
        return self.tool.filesystem.join(self.mac_port.layout_tests_dir(), path)

    def _read(self, path):
        return self.tool.filesystem.read_text_file(self._expand(path))

    def _write(self, path, contents):
        self.tool.filesystem.write_text_file(self._expand(path), contents)

    def _zero_out_test_expectations(self):
        for port_name in self.tool.port_factory.all_port_names():
            port = self.tool.port_factory.get(port_name)
            for path in port.expectations_files():
                self._write(path, '')
        self.tool.filesystem.written_files = {}

    def _setup_mock_build_data(self):
        for builder in ['MOCK Win7', 'MOCK Win7 (dbg)', 'MOCK Mac10.11']:
            self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
                'tests': {
                    'userscripts': {
                        'first-test.html': {
                            'expected': 'PASS',
                            'actual': 'IMAGE+TEXT'
                        },
                        'second-test.html': {
                            'expected': 'FAIL',
                            'actual': 'IMAGE+TEXT'
                        }
                    }
                }
            }))


class TestCopyExistingBaselinesInternal(BaseTestCase):
    command_constructor = CopyExistingBaselinesInternal

    def setUp(self):
        super(TestCopyExistingBaselinesInternal, self).setUp()

    def options(self, **kwargs):
        options_dict = {
            'results_directory': None,
            'suffixes': 'txt',
            'verbose': False,
        }
        options_dict.update(kwargs)
        return optparse.Values(options_dict)

    def baseline_path(self, path_from_layout_test_dir):
        port = self.tool.port_factory.get()
        return self.tool.filesystem.join(port.layout_tests_dir(), path_from_layout_test_dir)

    # The tests in this class all depend on the fall-back path graph
    # that is set up in |TestPort.FALLBACK_PATHS|.

    def test_copy_baseline_mac_newer_to_older_version(self):
        # The test-mac-mac10.11 baseline is copied over to the test-mac-mac10.10
        # baseline directory because test-mac-mac10.10 is the "immediate
        # predecessor" in the fall-back graph.
        self._write(
            self.baseline_path('platform/test-mac-mac10.11/failures/expected/image-expected.txt'),
            'original test-mac-mac10.11 result')
        self.assertFalse(self.tool.filesystem.exists(
            self.baseline_path('platform/test-mac-mac10.10/failures/expected/image-expected.txt')))

        self.command.execute(self.options(builder='MOCK Mac10.11', test='failures/expected/image.html'), [], self.tool)

        self.assertEqual(
            self._read(self.baseline_path('platform/test-mac-mac10.11/failures/expected/image-expected.txt')),
            'original test-mac-mac10.11 result')
        self.assertEqual(
            self._read(self.baseline_path('platform/test-mac-mac10.10/failures/expected/image-expected.txt')),
            'original test-mac-mac10.11 result')

    def test_copy_baseline_win10_to_linux_trusty_and_win7(self):
        # The test-win-win10 baseline is copied over to the test-linux-trusty
        # and test-win-win7 baseline paths, since both of these are "immediate
        # predecessors".
        self._write(
            self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt'),
            'original test-win-win10 result')
        self.assertFalse(self.tool.filesystem.exists(
            self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')))

        self.command.execute(self.options(builder='MOCK Win10', test='failures/expected/image.html'), [], self.tool)

        self.assertEqual(
            self._read(self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt')),
            'original test-win-win10 result')
        self.assertEqual(
            self._read(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')),
            'original test-win-win10 result')
        self.assertEqual(
            self._read(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')),
            'original test-win-win10 result')

    def test_no_copy_existing_baseline(self):
        # If a baseline exists already for an "immediate prdecessor" baseline
        # directory, (e.g. test-linux-trusty), then no "immediate successor"
        # baselines (e.g. test-win-win10) are copied over.
        self._write(
            self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt'),
            'original test-win-win10 result')
        self._write(
            self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt'),
            'original test-linux-trusty result')

        self.command.execute(self.options(builder='MOCK Win10', test='failures/expected/image.html'), [], self.tool)

        self.assertEqual(
            self._read(self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt')),
            'original test-win-win10 result')
        self.assertEqual(
            self._read(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')),
            'original test-linux-trusty result')

    def test_no_copy_skipped_test(self):
        # If a test is skipped on some platform, no baselines are copied over
        # to that directory. In this example, the test is skipped on linux,
        # so the test-win-win10 baseline is not copied over.
        port = self.tool.port_factory.get('test-win-win10')
        self._write(
            self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt'),
            'original test-win-win10 result')
        self._write(
            port.path_to_generic_test_expectations_file(),
            ('[ Win ] failures/expected/image.html [ Failure ]\n'
             '[ Linux ] failures/expected/image.html [ Skip ]\n'))

        self.command.execute(self.options(builder='MOCK Win10', test='failures/expected/image.html'), [], self.tool)

        self.assertFalse(
            self.tool.filesystem.exists(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')))

    def test_port_for_primary_baseline(self):
        self.assertEqual(self.command._port_for_primary_baseline('test-linux-trusty').name(), 'test-linux-trusty')
        self.assertEqual(self.command._port_for_primary_baseline('test-mac-mac10.11').name(), 'test-mac-mac10.11')

    def test_port_for_primary_baseline_not_found(self):
        with self.assertRaises(Exception):
            self.command._port_for_primary_baseline('test-foo-foo4.7')


class TestRebaselineTest(BaseTestCase):
    command_constructor = RebaselineTest  # AKA webkit-patch rebaseline-test-internal

    def setUp(self):
        super(TestRebaselineTest, self).setUp()

    @staticmethod
    def options(**kwargs):
        return optparse.Values(dict({
            'builder': 'MOCK Mac10.11',
            'test': 'userscripts/another-test.html',
            'suffixes': 'txt',
            'results_directory': None,
            'build_number': None
        }, **kwargs))

    def test_baseline_directory(self):
        command = self.command
        self.assertMultiLineEqual(command._baseline_directory('MOCK Mac10.11'),
                                  '/test.checkout/LayoutTests/platform/test-mac-mac10.11')
        self.assertMultiLineEqual(command._baseline_directory('MOCK Mac10.10'),
                                  '/test.checkout/LayoutTests/platform/test-mac-mac10.10')
        self.assertMultiLineEqual(command._baseline_directory('MOCK Trusty'),
                                  '/test.checkout/LayoutTests/platform/test-linux-trusty')
        self.assertMultiLineEqual(command._baseline_directory('MOCK Precise'),
                                  '/test.checkout/LayoutTests/platform/test-linux-precise')

    def test_rebaseline_updates_expectations_file_noop(self):
        self._zero_out_test_expectations()
        self._write(
            self.mac_expectations_path,
            ('Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]\n'
             'Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]\n'))
        self._write('fast/dom/Window/window-postmessage-clone-really-deep-array.html', 'Dummy test contents')
        self._write('fast/css/large-list-of-rules-crash.html', 'Dummy test contents')
        self._write('userscripts/another-test.html', 'Dummy test contents')

        self.command._rebaseline_test_and_update_expectations(self.options(suffixes='png,wav,txt'))

        self.assertItemsEqual(self.tool.web.urls_fetched,
                              [self.WEB_PREFIX + '/userscripts/another-test-actual.png',
                               self.WEB_PREFIX + '/userscripts/another-test-actual.wav',
                               self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
        new_expectations = self._read(self.mac_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]\n'
             'Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]\n'))

    def test_rebaseline_test(self):
        self.command._rebaseline_test('MOCK Trusty', 'userscripts/another-test.html', 'txt', self.WEB_PREFIX)
        self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])

    def test_rebaseline_test_with_results_directory(self):
        self._write('userscripts/another-test.html', 'test data')
        self._write(
            self.mac_expectations_path,
            ('Bug(x) [ Mac ] userscripts/another-test.html [ Failure ]\n'
             'bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n'))
        self.command._rebaseline_test_and_update_expectations(self.options(results_directory='/tmp'))
        self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscripts/another-test-actual.txt'])

    def test_rebaseline_reftest(self):
        self._write('userscripts/another-test.html', 'test data')
        self._write('userscripts/another-test-expected.html', 'generic result')
        OutputCapture().assert_outputs(
            self, self.command._rebaseline_test_and_update_expectations, args=[self.options(suffixes='png')],
            expected_logs='Cannot rebaseline image result for reftest: userscripts/another-test.html\n')
        self.assertDictEqual(self.command.expectation_line_changes.to_dict(), {'remove-lines': []})

    def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):
        self.tool.executive = MockExecutive()

        port = self.tool.port_factory.get('test-win-win7')
        self._write(
            port.host.filesystem.join(
                port.layout_tests_dir(),
                'platform/test-win-win10/failures/expected/image-expected.txt'),
            'original win10 result')

        oc = OutputCapture()
        try:
            options = optparse.Values({
                'optimize': True,
                'builder': 'MOCK Win10',
                'suffixes': 'txt',
                'verbose': True,
                'test': 'failures/expected/image.html',
                'results_directory': None,
                'build_number': None
            })
            oc.capture_output()
            self.command.execute(options, [], self.tool)
        finally:
            out, _, _ = oc.restore_output()

        self.assertMultiLineEqual(
            self._read(self.tool.filesystem.join(
                port.layout_tests_dir(),
                'platform/test-win-win10/failures/expected/image-expected.txt')),
            'MOCK Web result, convert 404 to None=True')
        self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(
            port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')))
        self.assertMultiLineEqual(
            out, '{"remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win10"}]}\n')


class TestAbstractParallelRebaselineCommand(BaseTestCase):
    command_constructor = AbstractParallelRebaselineCommand

    def test_builders_to_fetch_from(self):
        builders_to_fetch = self.command._builders_to_fetch_from(
            ['MOCK Win10', 'MOCK Win7 (dbg)(1)', 'MOCK Win7 (dbg)(2)', 'MOCK Win7'])
        self.assertEqual(builders_to_fetch, ['MOCK Win7', 'MOCK Win10'])

    def test_all_baseline_paths(self):
        test_prefix_list = {
            'passes/text.html': {
                Build('MOCK Win7'): ('txt', 'png'),
                Build('MOCK Win10'): ('txt',),
            }
        }
        # pylint: disable=protected-access
        baseline_paths = self.command._all_baseline_paths(test_prefix_list)
        self.assertEqual(baseline_paths, [
            '/test.checkout/LayoutTests/passes/text-expected.png',
            '/test.checkout/LayoutTests/passes/text-expected.txt',
            '/test.checkout/LayoutTests/platform/test-win-win10/passes/text-expected.txt',
            '/test.checkout/LayoutTests/platform/test-win-win7/passes/text-expected.png',
            '/test.checkout/LayoutTests/platform/test-win-win7/passes/text-expected.txt',
        ])

    def test_remove_all_pass_testharness_baselines(self):
        self.tool.filesystem.write_text_file(
            '/test.checkout/LayoutTests/passes/text-expected.txt',
            ('This is a testharness.js-based test.\n'
             'PASS: foo\n'
             'Harness: the test ran to completion.\n'))
        test_prefix_list = {
            'passes/text.html': {
                Build('MOCK Win7'): ('txt', 'png'),
                Build('MOCK Win10'): ('txt',),
            }
        }
        self.command._remove_all_pass_testharness_baselines(test_prefix_list)
        self.assertFalse(self.tool.filesystem.exists(
            '/test.checkout/LayoutTests/passes/text-expected.txt'))


class TestRebaselineJson(BaseTestCase):
    command_constructor = RebaselineJson

    def setUp(self):
        super(TestRebaselineJson, self).setUp()
        self.tool.executive = MockExecutive()

    def tearDown(self):
        super(TestRebaselineJson, self).tearDown()

    @staticmethod
    def options(**kwargs):
        return optparse.Values(dict({
            'optimize': True,
            'verbose': True,
            'results_directory': None
        }, **kwargs))

    def test_rebaseline_test_passes_on_all_builders(self):
        self._setup_mock_build_data()

        self.tool.buildbot.set_results(Build('MOCK Win7'), LayoutTestResults({
            'tests': {
                'userscripts': {
                    'first-test.html': {
                        'expected': 'NEEDSREBASELINE',
                        'actual': 'PASS'
                    }
                }
            }
        }))

        self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self.command.rebaseline(self.options(), {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}})

        self.assertEqual(self.tool.executive.calls, [])

    def test_rebaseline_all(self):
        self._setup_mock_build_data()

        self._write('userscripts/first-test.html', 'Dummy test contents')
        self.command.rebaseline(self.options(), {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}})

        # Note that we have one run_in_parallel() call followed by a run_command()
        self.assertEqual(
            self.tool.executive.calls,
            [
                [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']],
                [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']],
                [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png',
                  'userscripts/first-test.html', '--verbose']]
            ])

    def test_rebaseline_debug(self):
        self._setup_mock_build_data()

        self._write('userscripts/first-test.html', 'Dummy test contents')
        self.command.rebaseline(self.options(), {'userscripts/first-test.html': {Build('MOCK Win7 (dbg)'): ['txt', 'png']}})

        # Note that we have one run_in_parallel() call followed by a run_command()
        self.assertEqual(
            self.tool.executive.calls,
            [
                [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '--verbose']],
                [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder',
                  'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '--verbose']],
                [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png',
                  'userscripts/first-test.html', '--verbose']]
            ])

    def test_no_optimize(self):
        self._setup_mock_build_data()
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self.command.rebaseline(
            self.options(optimize=False),
            {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}})

        # Note that we have only one run_in_parallel() call
        self.assertEqual(
            self.tool.executive.calls,
            [
                [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']],
                [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']]
            ])

    def test_results_directory(self):
        self._setup_mock_build_data()
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self.command.rebaseline(
            self.options(optimize=False, results_directory='/tmp'),
            {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}})

        # Note that we have only one run_in_parallel() call
        self.assertEqual(
            self.tool.executive.calls,
            [
                [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']],
                [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose', '--results-directory', '/tmp']]
            ])

    def test_unstaged_baselines(self):
        git = self.tool.git()
        git.unstaged_changes = lambda: {
            'third_party/WebKit/LayoutTests/x/foo-expected.txt': 'M',
            'third_party/WebKit/LayoutTests/x/foo-expected.something': '?',
            'third_party/WebKit/LayoutTests/x/foo-expected.png': '?',
            'third_party/WebKit/LayoutTests/x/foo.html': 'M',
            'docs/something.md': '?',
        }
        self.assertEqual(
            self.command.unstaged_baselines(),
            [
                '/mock-checkout/third_party/WebKit/LayoutTests/x/foo-expected.png',
                '/mock-checkout/third_party/WebKit/LayoutTests/x/foo-expected.txt',
            ])


class TestRebaselineJsonUpdatesExpectationsFiles(BaseTestCase):
    command_constructor = RebaselineJson

    def setUp(self):
        super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp()
        self.tool.executive = MockExecutive()

        def mock_run_command(*args, **kwargs):  # pylint: disable=unused-argument
            return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "builder": "MOCK Mac10.11"}]}\n'
        self.tool.executive.run_command = mock_run_command

    @staticmethod
    def options():
        return optparse.Values({
            'optimize': False,
            'verbose': True,
            'results_directory': None
        })

    def test_rebaseline_updates_expectations_file(self):
        self._write(
            self.mac_expectations_path,
            ('Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n'
             'bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n'))
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self._setup_mock_build_data()

        self.command.rebaseline(
            self.options(),
            {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}})

        new_expectations = self._read(self.mac_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
             'bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n'))

    def test_rebaseline_updates_expectations_file_all_platforms(self):
        self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self._setup_mock_build_data()
        self.command.rebaseline(
            self.options(),
            {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}})
        new_expectations = self._read(self.mac_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n')

    def test_rebaseline_handles_platform_skips(self):
        # This test is just like test_rebaseline_updates_expectations_file_all_platforms(),
        # except that if a particular port happens to SKIP a test in an overrides file,
        # we count that as passing, and do not think that we still need to rebaseline it.
        self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        self._write('NeverFixTests', 'Bug(y) [ Android ] userscripts [ WontFix ]\n')
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self._setup_mock_build_data()

        self.command.rebaseline(
            self.options(),
            {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}})

        new_expectations = self._read(self.mac_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n')

    def test_rebaseline_handles_skips_in_file(self):
        # This test is like test_Rebaseline_handles_platform_skips, except that the
        # Skip is in the same (generic) file rather than a platform file. In this case,
        # the Skip line should be left unmodified. Note that the first line is now
        # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with
        # the second line.
        self._write(self.mac_expectations_path,
                    ('Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Failure ]\n'
                     'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n'))
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self._setup_mock_build_data()

        self.command.rebaseline(
            self.options(),
            {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}})

        new_expectations = self._read(self.mac_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n'
             'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n'))

    def test_rebaseline_handles_smoke_tests(self):
        # This test is just like test_rebaseline_handles_platform_skips, except that we check for
        # a test not being in the SmokeTests file, instead of using overrides files.
        # If a test is not part of the smoke tests, we count that as passing on ports that only
        # run smoke tests, and do not think that we still need to rebaseline it.
        self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        self._write('SmokeTests', 'fast/html/article-element.html')
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self._setup_mock_build_data()

        self.command.rebaseline(
            self.options(),
            {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}})

        new_expectations = self._read(self.mac_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n')


class TestRebaseline(BaseTestCase):
    # This command shares most of its logic with RebaselineJson, so these tests just test what is different.

    command_constructor = Rebaseline  # AKA webkit-patch rebaseline

    def test_rebaseline(self):
        self.command._builders_to_pull_from = lambda: ['MOCK Win7']

        self._write('userscripts/first-test.html', 'test data')

        self._zero_out_test_expectations()
        self._setup_mock_build_data()
        options = optparse.Values({
            'results_directory': False,
            'optimize': False,
            'builders': None,
            'suffixes': 'txt,png',
            'verbose': True
        })
        self.command.execute(options, ['userscripts/first-test.html'], self.tool)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']],
                [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
                  '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']]
            ])

    def test_rebaseline_directory(self):
        self.command._builders_to_pull_from = lambda: ['MOCK Win7']

        self._write('userscripts/first-test.html', 'test data')
        self._write('userscripts/second-test.html', 'test data')

        self._setup_mock_build_data()
        options = optparse.Values({
            'results_directory': False,
            'optimize': False,
            'builders': None,
            'suffixes': 'txt,png',
            'verbose': True
        })
        self.command.execute(options, ['userscripts'], self.tool)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [
                    ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                     '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose'],
                    ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                     '--builder', 'MOCK Win7', '--test', 'userscripts/second-test.html', '--verbose']
                ],
                [
                    ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
                     '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose'],
                    ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
                     '--builder', 'MOCK Win7', '--test', 'userscripts/second-test.html', '--verbose']
                ]
            ])


class TestRebaselineExpectations(BaseTestCase):
    command_constructor = RebaselineExpectations

    def setUp(self):
        super(TestRebaselineExpectations, self).setUp()

    @staticmethod
    def options():
        return optparse.Values({
            'optimize': False,
            'builders': None,
            'suffixes': ['txt'],
            'verbose': False,
            'platform': None,
            'results_directory': None
        })

    def _write_test_file(self, port, path, contents):
        abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path)
        self.tool.filesystem.write_text_file(abs_path, contents)

    def test_rebaseline_expectations(self):
        self._zero_out_test_expectations()

        self.tool.executive = MockExecutive()

        for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
            self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
                'tests': {
                    'userscripts': {
                        'another-test.html': {
                            'expected': 'PASS',
                            'actual': 'PASS TEXT'
                        },
                        'images.svg': {
                            'expected': 'FAIL',
                            'actual': 'IMAGE+TEXT'
                        }
                    }
                }
            }))

        self._write('userscripts/another-test.html', 'Dummy test contents')
        self._write('userscripts/images.svg', 'Dummy test contents')
        self.command._tests_to_rebaseline = lambda port: {
            'userscripts/another-test.html': set(['txt']),
            'userscripts/images.svg': set(['png']),
            'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']),
        }

        self.command.execute(self.options(), [], self.tool)

        self.assertEqual(self.tool.executive.calls, [
            [
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/another-test.html'],
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/another-test.html'],
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png',
                 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/images.svg'],
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png',
                 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/images.svg'],
            ],
            [
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/another-test.html'],
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/another-test.html'],
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png',
                 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/images.svg'],
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png',
                 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/images.svg'],
            ],
        ])

    def test_rebaseline_expectations_reftests(self):
        self._zero_out_test_expectations()

        self.tool.executive = MockExecutive()

        for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
            self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
                'tests': {
                    'userscripts': {
                        'reftest-text.html': {
                            'expected': 'PASS',
                            'actual': 'TEXT'
                        },
                        'reftest-image.html': {
                            'expected': 'FAIL',
                            'actual': 'IMAGE'
                        },
                        'reftest-image-text.html': {
                            'expected': 'FAIL',
                            'actual': 'IMAGE+TEXT'
                        }
                    }
                }
            }))

        self._write('userscripts/reftest-text.html', 'Dummy test contents')
        self._write('userscripts/reftest-text-expected.html', 'Dummy test contents')
        self._write('userscripts/reftest-text-expected.html', 'Dummy test contents')
        self.command._tests_to_rebaseline = lambda port: {
            'userscripts/reftest-text.html': set(['txt']),
            'userscripts/reftest-image.html': set(['png']),
            'userscripts/reftest-image-text.html': set(['png', 'txt']),
        }

        self.command.execute(self.options(), [], self.tool)

        self.assertEqual(self.tool.executive.calls, [
            [
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-text.html'],
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-text.html'],
            ],
            [
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-text.html'],
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-text.html'],
            ],
        ])

    def test_rebaseline_expectations_noop(self):
        self._zero_out_test_expectations()

        oc = OutputCapture()
        try:
            oc.capture_output()
            self.command.execute(self.options(), [], self.tool)
        finally:
            _, _, logs = oc.restore_output()
            self.assertEqual(self.tool.filesystem.written_files, {})
            self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n')

    def disabled_test_overrides_are_included_correctly(self):
        # TODO(qyearsley): Fix or remove this test method.
        # This tests that any tests marked as REBASELINE in the overrides are found, but
        # that the overrides do not get written into the main file.
        self._zero_out_test_expectations()

        self._write(self.mac_expectations_path, '')
        self.mac_port.expectations_dict = lambda: {
            self.mac_expectations_path: '',
            'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n'
                          'Bug(y) userscripts/test.html [ Crash ]\n')}
        self._write('/userscripts/another-test.html', '')

        self.assertDictEqual(self.command._tests_to_rebaseline(self.mac_port),
                             {'userscripts/another-test.html': set(['png', 'txt', 'wav'])})
        self.assertEqual(self._read(self.mac_expectations_path), '')

    def test_rebaseline_without_other_expectations(self):
        self._write('userscripts/another-test.html', 'Dummy test contents')
        self._write(self.mac_expectations_path, 'Bug(x) userscripts/another-test.html [ Rebaseline ]\n')
        self.assertDictEqual(self.command._tests_to_rebaseline(self.mac_port),
                             {'userscripts/another-test.html': ('png', 'wav', 'txt')})

    def test_rebaseline_test_passes_everywhere(self):
        test_port = self.tool.port_factory.get('test')

        for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
            self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
                'tests': {
                    'fast': {
                        'dom': {
                            'prototype-taco.html': {
                                'expected': 'FAIL',
                                'actual': 'PASS',
                                'is_unexpected': True
                            }
                        }
                    }
                }
            }))

        self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ]
""")

        self._write_test_file(test_port, 'fast/dom/prototype-taco.html', 'Dummy test contents')

        self.tool.executive = MockLineRemovingExecutive()

        self.tool.builders = BuilderList({
            'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']},
            'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
        })

        self.command.execute(self.options(), [], self.tool)
        self.assertEqual(self.tool.executive.calls, [])

        # The mac ports should both be removed since they're the only ones in the builder list.
        self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ]
""")

    def test_rebaseline_missing(self):
        self.tool.buildbot.set_results(Build('MOCK Mac10.10'), LayoutTestResults({
            'tests': {
                'fast': {
                    'dom': {
                        'missing-text.html': {
                            'expected': 'PASS',
                            'actual': 'MISSING',
                            'is_unexpected': True,
                            'is_missing_text': True
                        },
                        'missing-text-and-image.html': {
                            'expected': 'PASS',
                            'actual': 'MISSING',
                            'is_unexpected': True,
                            'is_missing_text': True,
                            'is_missing_image': True
                        },
                        'missing-image.html': {
                            'expected': 'PASS',
                            'actual': 'MISSING',
                            'is_unexpected': True,
                            'is_missing_image': True
                        }
                    }
                }
            }
        }))

        self._write('fast/dom/missing-text.html', 'Dummy test contents')
        self._write('fast/dom/missing-text-and-image.html', 'Dummy test contents')
        self._write('fast/dom/missing-image.html', 'Dummy test contents')

        self.command._tests_to_rebaseline = lambda port: {
            'fast/dom/missing-text.html': set(['txt', 'png']),
            'fast/dom/missing-text-and-image.html': set(['txt', 'png']),
            'fast/dom/missing-image.html': set(['txt', 'png']),
        }

        self.command.execute(self.options(), [], self.tool)

        self.assertEqual(self.tool.executive.calls, [
            [
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.html'],
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png',
                 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-and-image.html'],
                ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png',
                 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image.html'],
            ],
            [
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt',
                 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.html'],
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
                 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-and-image.html'],
                ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png',
                 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image.html'],
            ]
        ])


class MockLineRemovingExecutive(MockExecutive):

    def run_in_parallel(self, commands):
        assert len(commands)

        num_previous_calls = len(self.calls)
        command_outputs = []
        for cmd_line, cwd in commands:
            out = self.run_command(cmd_line, cwd=cwd)
            if 'rebaseline-test-internal' in cmd_line:
                out = '{"remove-lines": [{"test": "%s", "builder": "%s"}]}\n' % (cmd_line[8], cmd_line[6])
            command_outputs.append([0, out, ''])

        new_calls = self.calls[num_previous_calls:]
        self.calls = self.calls[:num_previous_calls]
        self.calls.append(new_calls)
        return command_outputs