| # Copyright 2016 The Chromium Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| import optparse | |
| import unittest | |
| from webkitpy.common.net.buildbot import Build | |
| from webkitpy.common.net.layout_test_results import LayoutTestResults | |
| from webkitpy.common.system.executive_mock import MockExecutive | |
| from webkitpy.common.system.output_capture import OutputCapture | |
| from webkitpy.layout_tests.builder_list import BuilderList | |
| from webkitpy.tool.commands.rebaseline import ( | |
| AbstractParallelRebaselineCommand, CopyExistingBaselinesInternal, | |
| Rebaseline, RebaselineExpectations, RebaselineJson, RebaselineTest | |
| ) | |
| from webkitpy.tool.mock_tool import MockWebKitPatch | |
| # pylint: disable=protected-access | |
| class BaseTestCase(unittest.TestCase): | |
| WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/MOCK_Mac10_11/results/layout-test-results' | |
| command_constructor = None | |
| def setUp(self): | |
| self.tool = MockWebKitPatch() | |
| # Lint warns that command_constructor might not be set, but this is intentional; pylint: disable=not-callable | |
| self.command = self.command_constructor() | |
| self.command._tool = self.tool | |
| self.tool.builders = BuilderList({ | |
| 'MOCK Mac10.10 (dbg)': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Debug']}, | |
| 'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']}, | |
| 'MOCK Mac10.11 (dbg)': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Debug']}, | |
| 'MOCK Mac10.11 ASAN': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']}, | |
| 'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']}, | |
| 'MOCK Precise': {'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release']}, | |
| 'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']}, | |
| 'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']}, | |
| 'MOCK Win7 (dbg)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']}, | |
| 'MOCK Win7 (dbg)(1)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']}, | |
| 'MOCK Win7 (dbg)(2)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']}, | |
| 'MOCK Win7': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release']}, | |
| }) | |
| self.mac_port = self.tool.port_factory.get_from_builder_name('MOCK Mac10.11') | |
| self.mac_expectations_path = self.mac_port.path_to_generic_test_expectations_file() | |
| self.tool.filesystem.write_text_file( | |
| self.tool.filesystem.join(self.mac_port.layout_tests_dir(), 'VirtualTestSuites'), '[]') | |
| # In AbstractParallelRebaselineCommand._rebaseline_commands, a default port | |
| # object is gotten using self.tool.port_factory.get(), which is used to get | |
| # test paths -- and the layout tests directory may be different for the "test" | |
| # ports and real ports. Since only "test" ports are used in this class, | |
| # we can make the default port also a "test" port. | |
| self.original_port_factory_get = self.tool.port_factory.get | |
| test_port = self.tool.port_factory.get('test') | |
| def get_test_port(port_name=None, options=None, **kwargs): | |
| if not port_name: | |
| return test_port | |
| return self.original_port_factory_get(port_name, options, **kwargs) | |
| self.tool.port_factory.get = get_test_port | |
| def tearDown(self): | |
| self.tool.port_factory.get = self.original_port_factory_get | |
| def _expand(self, path): | |
| if self.tool.filesystem.isabs(path): | |
| return path | |
| return self.tool.filesystem.join(self.mac_port.layout_tests_dir(), path) | |
| def _read(self, path): | |
| return self.tool.filesystem.read_text_file(self._expand(path)) | |
| def _write(self, path, contents): | |
| self.tool.filesystem.write_text_file(self._expand(path), contents) | |
| def _zero_out_test_expectations(self): | |
| for port_name in self.tool.port_factory.all_port_names(): | |
| port = self.tool.port_factory.get(port_name) | |
| for path in port.expectations_files(): | |
| self._write(path, '') | |
| self.tool.filesystem.written_files = {} | |
| def _setup_mock_build_data(self): | |
| for builder in ['MOCK Win7', 'MOCK Win7 (dbg)', 'MOCK Mac10.11']: | |
| self.tool.buildbot.set_results(Build(builder), LayoutTestResults({ | |
| 'tests': { | |
| 'userscripts': { | |
| 'first-test.html': { | |
| 'expected': 'PASS', | |
| 'actual': 'IMAGE+TEXT' | |
| }, | |
| 'second-test.html': { | |
| 'expected': 'FAIL', | |
| 'actual': 'IMAGE+TEXT' | |
| } | |
| } | |
| } | |
| })) | |
| class TestCopyExistingBaselinesInternal(BaseTestCase): | |
| command_constructor = CopyExistingBaselinesInternal | |
| def setUp(self): | |
| super(TestCopyExistingBaselinesInternal, self).setUp() | |
| def options(self, **kwargs): | |
| options_dict = { | |
| 'results_directory': None, | |
| 'suffixes': 'txt', | |
| 'verbose': False, | |
| } | |
| options_dict.update(kwargs) | |
| return optparse.Values(options_dict) | |
| def baseline_path(self, path_from_layout_test_dir): | |
| port = self.tool.port_factory.get() | |
| return self.tool.filesystem.join(port.layout_tests_dir(), path_from_layout_test_dir) | |
| # The tests in this class all depend on the fall-back path graph | |
| # that is set up in |TestPort.FALLBACK_PATHS|. | |
| def test_copy_baseline_mac_newer_to_older_version(self): | |
| # The test-mac-mac10.11 baseline is copied over to the test-mac-mac10.10 | |
| # baseline directory because test-mac-mac10.10 is the "immediate | |
| # predecessor" in the fall-back graph. | |
| self._write( | |
| self.baseline_path('platform/test-mac-mac10.11/failures/expected/image-expected.txt'), | |
| 'original test-mac-mac10.11 result') | |
| self.assertFalse(self.tool.filesystem.exists( | |
| self.baseline_path('platform/test-mac-mac10.10/failures/expected/image-expected.txt'))) | |
| self.command.execute(self.options(builder='MOCK Mac10.11', test='failures/expected/image.html'), [], self.tool) | |
| self.assertEqual( | |
| self._read(self.baseline_path('platform/test-mac-mac10.11/failures/expected/image-expected.txt')), | |
| 'original test-mac-mac10.11 result') | |
| self.assertEqual( | |
| self._read(self.baseline_path('platform/test-mac-mac10.10/failures/expected/image-expected.txt')), | |
| 'original test-mac-mac10.11 result') | |
| def test_copy_baseline_win10_to_linux_trusty_and_win7(self): | |
| # The test-win-win10 baseline is copied over to the test-linux-trusty | |
| # and test-win-win7 baseline paths, since both of these are "immediate | |
| # predecessors". | |
| self._write( | |
| self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt'), | |
| 'original test-win-win10 result') | |
| self.assertFalse(self.tool.filesystem.exists( | |
| self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt'))) | |
| self.command.execute(self.options(builder='MOCK Win10', test='failures/expected/image.html'), [], self.tool) | |
| self.assertEqual( | |
| self._read(self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt')), | |
| 'original test-win-win10 result') | |
| self.assertEqual( | |
| self._read(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')), | |
| 'original test-win-win10 result') | |
| self.assertEqual( | |
| self._read(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')), | |
| 'original test-win-win10 result') | |
| def test_no_copy_existing_baseline(self): | |
| # If a baseline exists already for an "immediate prdecessor" baseline | |
| # directory, (e.g. test-linux-trusty), then no "immediate successor" | |
| # baselines (e.g. test-win-win10) are copied over. | |
| self._write( | |
| self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt'), | |
| 'original test-win-win10 result') | |
| self._write( | |
| self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt'), | |
| 'original test-linux-trusty result') | |
| self.command.execute(self.options(builder='MOCK Win10', test='failures/expected/image.html'), [], self.tool) | |
| self.assertEqual( | |
| self._read(self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt')), | |
| 'original test-win-win10 result') | |
| self.assertEqual( | |
| self._read(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt')), | |
| 'original test-linux-trusty result') | |
| def test_no_copy_skipped_test(self): | |
| # If a test is skipped on some platform, no baselines are copied over | |
| # to that directory. In this example, the test is skipped on linux, | |
| # so the test-win-win10 baseline is not copied over. | |
| port = self.tool.port_factory.get('test-win-win10') | |
| self._write( | |
| self.baseline_path('platform/test-win-win10/failures/expected/image-expected.txt'), | |
| 'original test-win-win10 result') | |
| self._write( | |
| port.path_to_generic_test_expectations_file(), | |
| ('[ Win ] failures/expected/image.html [ Failure ]\n' | |
| '[ Linux ] failures/expected/image.html [ Skip ]\n')) | |
| self.command.execute(self.options(builder='MOCK Win10', test='failures/expected/image.html'), [], self.tool) | |
| self.assertFalse( | |
| self.tool.filesystem.exists(self.baseline_path('platform/test-linux-trusty/failures/expected/image-expected.txt'))) | |
| def test_port_for_primary_baseline(self): | |
| self.assertEqual(self.command._port_for_primary_baseline('test-linux-trusty').name(), 'test-linux-trusty') | |
| self.assertEqual(self.command._port_for_primary_baseline('test-mac-mac10.11').name(), 'test-mac-mac10.11') | |
| def test_port_for_primary_baseline_not_found(self): | |
| with self.assertRaises(Exception): | |
| self.command._port_for_primary_baseline('test-foo-foo4.7') | |
| class TestRebaselineTest(BaseTestCase): | |
| command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-internal | |
| def setUp(self): | |
| super(TestRebaselineTest, self).setUp() | |
| @staticmethod | |
| def options(**kwargs): | |
| return optparse.Values(dict({ | |
| 'builder': 'MOCK Mac10.11', | |
| 'test': 'userscripts/another-test.html', | |
| 'suffixes': 'txt', | |
| 'results_directory': None, | |
| 'build_number': None | |
| }, **kwargs)) | |
| def test_baseline_directory(self): | |
| command = self.command | |
| self.assertMultiLineEqual(command._baseline_directory('MOCK Mac10.11'), | |
| '/test.checkout/LayoutTests/platform/test-mac-mac10.11') | |
| self.assertMultiLineEqual(command._baseline_directory('MOCK Mac10.10'), | |
| '/test.checkout/LayoutTests/platform/test-mac-mac10.10') | |
| self.assertMultiLineEqual(command._baseline_directory('MOCK Trusty'), | |
| '/test.checkout/LayoutTests/platform/test-linux-trusty') | |
| self.assertMultiLineEqual(command._baseline_directory('MOCK Precise'), | |
| '/test.checkout/LayoutTests/platform/test-linux-precise') | |
| def test_rebaseline_updates_expectations_file_noop(self): | |
| self._zero_out_test_expectations() | |
| self._write( | |
| self.mac_expectations_path, | |
| ('Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]\n' | |
| 'Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]\n')) | |
| self._write('fast/dom/Window/window-postmessage-clone-really-deep-array.html', 'Dummy test contents') | |
| self._write('fast/css/large-list-of-rules-crash.html', 'Dummy test contents') | |
| self._write('userscripts/another-test.html', 'Dummy test contents') | |
| self.command._rebaseline_test_and_update_expectations(self.options(suffixes='png,wav,txt')) | |
| self.assertItemsEqual(self.tool.web.urls_fetched, | |
| [self.WEB_PREFIX + '/userscripts/another-test-actual.png', | |
| self.WEB_PREFIX + '/userscripts/another-test-actual.wav', | |
| self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) | |
| new_expectations = self._read(self.mac_expectations_path) | |
| self.assertMultiLineEqual( | |
| new_expectations, | |
| ('Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]\n' | |
| 'Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]\n')) | |
| def test_rebaseline_test(self): | |
| self.command._rebaseline_test('MOCK Trusty', 'userscripts/another-test.html', 'txt', self.WEB_PREFIX) | |
| self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) | |
| def test_rebaseline_test_with_results_directory(self): | |
| self._write('userscripts/another-test.html', 'test data') | |
| self._write( | |
| self.mac_expectations_path, | |
| ('Bug(x) [ Mac ] userscripts/another-test.html [ Failure ]\n' | |
| 'bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n')) | |
| self.command._rebaseline_test_and_update_expectations(self.options(results_directory='/tmp')) | |
| self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscripts/another-test-actual.txt']) | |
| def test_rebaseline_reftest(self): | |
| self._write('userscripts/another-test.html', 'test data') | |
| self._write('userscripts/another-test-expected.html', 'generic result') | |
| OutputCapture().assert_outputs( | |
| self, self.command._rebaseline_test_and_update_expectations, args=[self.options(suffixes='png')], | |
| expected_logs='Cannot rebaseline image result for reftest: userscripts/another-test.html\n') | |
| self.assertDictEqual(self.command.expectation_line_changes.to_dict(), {'remove-lines': []}) | |
| def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): | |
| self.tool.executive = MockExecutive() | |
| port = self.tool.port_factory.get('test-win-win7') | |
| self._write( | |
| port.host.filesystem.join( | |
| port.layout_tests_dir(), | |
| 'platform/test-win-win10/failures/expected/image-expected.txt'), | |
| 'original win10 result') | |
| oc = OutputCapture() | |
| try: | |
| options = optparse.Values({ | |
| 'optimize': True, | |
| 'builder': 'MOCK Win10', | |
| 'suffixes': 'txt', | |
| 'verbose': True, | |
| 'test': 'failures/expected/image.html', | |
| 'results_directory': None, | |
| 'build_number': None | |
| }) | |
| oc.capture_output() | |
| self.command.execute(options, [], self.tool) | |
| finally: | |
| out, _, _ = oc.restore_output() | |
| self.assertMultiLineEqual( | |
| self._read(self.tool.filesystem.join( | |
| port.layout_tests_dir(), | |
| 'platform/test-win-win10/failures/expected/image-expected.txt')), | |
| 'MOCK Web result, convert 404 to None=True') | |
| self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | |
| port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'))) | |
| self.assertMultiLineEqual( | |
| out, '{"remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win10"}]}\n') | |
| class TestAbstractParallelRebaselineCommand(BaseTestCase): | |
| command_constructor = AbstractParallelRebaselineCommand | |
| def test_builders_to_fetch_from(self): | |
| builders_to_fetch = self.command._builders_to_fetch_from( | |
| ['MOCK Win10', 'MOCK Win7 (dbg)(1)', 'MOCK Win7 (dbg)(2)', 'MOCK Win7']) | |
| self.assertEqual(builders_to_fetch, ['MOCK Win7', 'MOCK Win10']) | |
| def test_all_baseline_paths(self): | |
| test_prefix_list = { | |
| 'passes/text.html': { | |
| Build('MOCK Win7'): ('txt', 'png'), | |
| Build('MOCK Win10'): ('txt',), | |
| } | |
| } | |
| # pylint: disable=protected-access | |
| baseline_paths = self.command._all_baseline_paths(test_prefix_list) | |
| self.assertEqual(baseline_paths, [ | |
| '/test.checkout/LayoutTests/passes/text-expected.png', | |
| '/test.checkout/LayoutTests/passes/text-expected.txt', | |
| '/test.checkout/LayoutTests/platform/test-win-win10/passes/text-expected.txt', | |
| '/test.checkout/LayoutTests/platform/test-win-win7/passes/text-expected.png', | |
| '/test.checkout/LayoutTests/platform/test-win-win7/passes/text-expected.txt', | |
| ]) | |
| def test_remove_all_pass_testharness_baselines(self): | |
| self.tool.filesystem.write_text_file( | |
| '/test.checkout/LayoutTests/passes/text-expected.txt', | |
| ('This is a testharness.js-based test.\n' | |
| 'PASS: foo\n' | |
| 'Harness: the test ran to completion.\n')) | |
| test_prefix_list = { | |
| 'passes/text.html': { | |
| Build('MOCK Win7'): ('txt', 'png'), | |
| Build('MOCK Win10'): ('txt',), | |
| } | |
| } | |
| self.command._remove_all_pass_testharness_baselines(test_prefix_list) | |
| self.assertFalse(self.tool.filesystem.exists( | |
| '/test.checkout/LayoutTests/passes/text-expected.txt')) | |
| class TestRebaselineJson(BaseTestCase): | |
| command_constructor = RebaselineJson | |
| def setUp(self): | |
| super(TestRebaselineJson, self).setUp() | |
| self.tool.executive = MockExecutive() | |
| def tearDown(self): | |
| super(TestRebaselineJson, self).tearDown() | |
| @staticmethod | |
| def options(**kwargs): | |
| return optparse.Values(dict({ | |
| 'optimize': True, | |
| 'verbose': True, | |
| 'results_directory': None | |
| }, **kwargs)) | |
| def test_rebaseline_test_passes_on_all_builders(self): | |
| self._setup_mock_build_data() | |
| self.tool.buildbot.set_results(Build('MOCK Win7'), LayoutTestResults({ | |
| 'tests': { | |
| 'userscripts': { | |
| 'first-test.html': { | |
| 'expected': 'NEEDSREBASELINE', | |
| 'actual': 'PASS' | |
| } | |
| } | |
| } | |
| })) | |
| self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n') | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self.command.rebaseline(self.options(), {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}}) | |
| self.assertEqual(self.tool.executive.calls, []) | |
| def test_rebaseline_all(self): | |
| self._setup_mock_build_data() | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self.command.rebaseline(self.options(), {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}}) | |
| # Note that we have one run_in_parallel() call followed by a run_command() | |
| self.assertEqual( | |
| self.tool.executive.calls, | |
| [ | |
| [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']], | |
| [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']], | |
| [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png', | |
| 'userscripts/first-test.html', '--verbose']] | |
| ]) | |
| def test_rebaseline_debug(self): | |
| self._setup_mock_build_data() | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self.command.rebaseline(self.options(), {'userscripts/first-test.html': {Build('MOCK Win7 (dbg)'): ['txt', 'png']}}) | |
| # Note that we have one run_in_parallel() call followed by a run_command() | |
| self.assertEqual( | |
| self.tool.executive.calls, | |
| [ | |
| [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '--verbose']], | |
| [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', | |
| 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '--verbose']], | |
| [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png', | |
| 'userscripts/first-test.html', '--verbose']] | |
| ]) | |
| def test_no_optimize(self): | |
| self._setup_mock_build_data() | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self.command.rebaseline( | |
| self.options(optimize=False), | |
| {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}}) | |
| # Note that we have only one run_in_parallel() call | |
| self.assertEqual( | |
| self.tool.executive.calls, | |
| [ | |
| [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']], | |
| [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']] | |
| ]) | |
| def test_results_directory(self): | |
| self._setup_mock_build_data() | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self.command.rebaseline( | |
| self.options(optimize=False, results_directory='/tmp'), | |
| {'userscripts/first-test.html': {Build('MOCK Win7'): ['txt', 'png']}}) | |
| # Note that we have only one run_in_parallel() call | |
| self.assertEqual( | |
| self.tool.executive.calls, | |
| [ | |
| [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']], | |
| [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose', '--results-directory', '/tmp']] | |
| ]) | |
| def test_unstaged_baselines(self): | |
| git = self.tool.git() | |
| git.unstaged_changes = lambda: { | |
| 'third_party/WebKit/LayoutTests/x/foo-expected.txt': 'M', | |
| 'third_party/WebKit/LayoutTests/x/foo-expected.something': '?', | |
| 'third_party/WebKit/LayoutTests/x/foo-expected.png': '?', | |
| 'third_party/WebKit/LayoutTests/x/foo.html': 'M', | |
| 'docs/something.md': '?', | |
| } | |
| self.assertEqual( | |
| self.command.unstaged_baselines(), | |
| [ | |
| '/mock-checkout/third_party/WebKit/LayoutTests/x/foo-expected.png', | |
| '/mock-checkout/third_party/WebKit/LayoutTests/x/foo-expected.txt', | |
| ]) | |
| class TestRebaselineJsonUpdatesExpectationsFiles(BaseTestCase): | |
| command_constructor = RebaselineJson | |
| def setUp(self): | |
| super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp() | |
| self.tool.executive = MockExecutive() | |
| def mock_run_command(*args, **kwargs): # pylint: disable=unused-argument | |
| return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "builder": "MOCK Mac10.11"}]}\n' | |
| self.tool.executive.run_command = mock_run_command | |
| @staticmethod | |
| def options(): | |
| return optparse.Values({ | |
| 'optimize': False, | |
| 'verbose': True, | |
| 'results_directory': None | |
| }) | |
| def test_rebaseline_updates_expectations_file(self): | |
| self._write( | |
| self.mac_expectations_path, | |
| ('Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n' | |
| 'bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n')) | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self._setup_mock_build_data() | |
| self.command.rebaseline( | |
| self.options(), | |
| {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}}) | |
| new_expectations = self._read(self.mac_expectations_path) | |
| self.assertMultiLineEqual( | |
| new_expectations, | |
| ('Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n' | |
| 'bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n')) | |
| def test_rebaseline_updates_expectations_file_all_platforms(self): | |
| self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n') | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self._setup_mock_build_data() | |
| self.command.rebaseline( | |
| self.options(), | |
| {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}}) | |
| new_expectations = self._read(self.mac_expectations_path) | |
| self.assertMultiLineEqual( | |
| new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n') | |
| def test_rebaseline_handles_platform_skips(self): | |
| # This test is just like test_rebaseline_updates_expectations_file_all_platforms(), | |
| # except that if a particular port happens to SKIP a test in an overrides file, | |
| # we count that as passing, and do not think that we still need to rebaseline it. | |
| self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n') | |
| self._write('NeverFixTests', 'Bug(y) [ Android ] userscripts [ WontFix ]\n') | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self._setup_mock_build_data() | |
| self.command.rebaseline( | |
| self.options(), | |
| {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}}) | |
| new_expectations = self._read(self.mac_expectations_path) | |
| self.assertMultiLineEqual( | |
| new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n') | |
| def test_rebaseline_handles_skips_in_file(self): | |
| # This test is like test_Rebaseline_handles_platform_skips, except that the | |
| # Skip is in the same (generic) file rather than a platform file. In this case, | |
| # the Skip line should be left unmodified. Note that the first line is now | |
| # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with | |
| # the second line. | |
| self._write(self.mac_expectations_path, | |
| ('Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Failure ]\n' | |
| 'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n')) | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self._setup_mock_build_data() | |
| self.command.rebaseline( | |
| self.options(), | |
| {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}}) | |
| new_expectations = self._read(self.mac_expectations_path) | |
| self.assertMultiLineEqual( | |
| new_expectations, | |
| ('Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n' | |
| 'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n')) | |
| def test_rebaseline_handles_smoke_tests(self): | |
| # This test is just like test_rebaseline_handles_platform_skips, except that we check for | |
| # a test not being in the SmokeTests file, instead of using overrides files. | |
| # If a test is not part of the smoke tests, we count that as passing on ports that only | |
| # run smoke tests, and do not think that we still need to rebaseline it. | |
| self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n') | |
| self._write('SmokeTests', 'fast/html/article-element.html') | |
| self._write('userscripts/first-test.html', 'Dummy test contents') | |
| self._setup_mock_build_data() | |
| self.command.rebaseline( | |
| self.options(), | |
| {'userscripts/first-test.html': {Build('MOCK Mac10.11'): ['txt', 'png']}}) | |
| new_expectations = self._read(self.mac_expectations_path) | |
| self.assertMultiLineEqual( | |
| new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n') | |
| class TestRebaseline(BaseTestCase): | |
| # This command shares most of its logic with RebaselineJson, so these tests just test what is different. | |
| command_constructor = Rebaseline # AKA webkit-patch rebaseline | |
| def test_rebaseline(self): | |
| self.command._builders_to_pull_from = lambda: ['MOCK Win7'] | |
| self._write('userscripts/first-test.html', 'test data') | |
| self._zero_out_test_expectations() | |
| self._setup_mock_build_data() | |
| options = optparse.Values({ | |
| 'results_directory': False, | |
| 'optimize': False, | |
| 'builders': None, | |
| 'suffixes': 'txt,png', | |
| 'verbose': True | |
| }) | |
| self.command.execute(options, ['userscripts/first-test.html'], self.tool) | |
| self.assertEqual( | |
| self.tool.executive.calls, | |
| [ | |
| [['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']], | |
| [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose']] | |
| ]) | |
| def test_rebaseline_directory(self): | |
| self.command._builders_to_pull_from = lambda: ['MOCK Win7'] | |
| self._write('userscripts/first-test.html', 'test data') | |
| self._write('userscripts/second-test.html', 'test data') | |
| self._setup_mock_build_data() | |
| options = optparse.Values({ | |
| 'results_directory': False, | |
| 'optimize': False, | |
| 'builders': None, | |
| 'suffixes': 'txt,png', | |
| 'verbose': True | |
| }) | |
| self.command.execute(options, ['userscripts'], self.tool) | |
| self.assertEqual( | |
| self.tool.executive.calls, | |
| [ | |
| [ | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose'], | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/second-test.html', '--verbose'] | |
| ], | |
| [ | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose'], | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Win7', '--test', 'userscripts/second-test.html', '--verbose'] | |
| ] | |
| ]) | |
| class TestRebaselineExpectations(BaseTestCase): | |
| command_constructor = RebaselineExpectations | |
| def setUp(self): | |
| super(TestRebaselineExpectations, self).setUp() | |
| @staticmethod | |
| def options(): | |
| return optparse.Values({ | |
| 'optimize': False, | |
| 'builders': None, | |
| 'suffixes': ['txt'], | |
| 'verbose': False, | |
| 'platform': None, | |
| 'results_directory': None | |
| }) | |
| def _write_test_file(self, port, path, contents): | |
| abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) | |
| self.tool.filesystem.write_text_file(abs_path, contents) | |
| def test_rebaseline_expectations(self): | |
| self._zero_out_test_expectations() | |
| self.tool.executive = MockExecutive() | |
| for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']: | |
| self.tool.buildbot.set_results(Build(builder), LayoutTestResults({ | |
| 'tests': { | |
| 'userscripts': { | |
| 'another-test.html': { | |
| 'expected': 'PASS', | |
| 'actual': 'PASS TEXT' | |
| }, | |
| 'images.svg': { | |
| 'expected': 'FAIL', | |
| 'actual': 'IMAGE+TEXT' | |
| } | |
| } | |
| } | |
| })) | |
| self._write('userscripts/another-test.html', 'Dummy test contents') | |
| self._write('userscripts/images.svg', 'Dummy test contents') | |
| self.command._tests_to_rebaseline = lambda port: { | |
| 'userscripts/another-test.html': set(['txt']), | |
| 'userscripts/images.svg': set(['png']), | |
| 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']), | |
| } | |
| self.command.execute(self.options(), [], self.tool) | |
| self.assertEqual(self.tool.executive.calls, [ | |
| [ | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.10', '--test', 'userscripts/another-test.html'], | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.11', '--test', 'userscripts/another-test.html'], | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', | |
| '--builder', 'MOCK Mac10.10', '--test', 'userscripts/images.svg'], | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', | |
| '--builder', 'MOCK Mac10.11', '--test', 'userscripts/images.svg'], | |
| ], | |
| [ | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.10', '--test', 'userscripts/another-test.html'], | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.11', '--test', 'userscripts/another-test.html'], | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', | |
| '--builder', 'MOCK Mac10.10', '--test', 'userscripts/images.svg'], | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', | |
| '--builder', 'MOCK Mac10.11', '--test', 'userscripts/images.svg'], | |
| ], | |
| ]) | |
| def test_rebaseline_expectations_reftests(self): | |
| self._zero_out_test_expectations() | |
| self.tool.executive = MockExecutive() | |
| for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']: | |
| self.tool.buildbot.set_results(Build(builder), LayoutTestResults({ | |
| 'tests': { | |
| 'userscripts': { | |
| 'reftest-text.html': { | |
| 'expected': 'PASS', | |
| 'actual': 'TEXT' | |
| }, | |
| 'reftest-image.html': { | |
| 'expected': 'FAIL', | |
| 'actual': 'IMAGE' | |
| }, | |
| 'reftest-image-text.html': { | |
| 'expected': 'FAIL', | |
| 'actual': 'IMAGE+TEXT' | |
| } | |
| } | |
| } | |
| })) | |
| self._write('userscripts/reftest-text.html', 'Dummy test contents') | |
| self._write('userscripts/reftest-text-expected.html', 'Dummy test contents') | |
| self._write('userscripts/reftest-text-expected.html', 'Dummy test contents') | |
| self.command._tests_to_rebaseline = lambda port: { | |
| 'userscripts/reftest-text.html': set(['txt']), | |
| 'userscripts/reftest-image.html': set(['png']), | |
| 'userscripts/reftest-image-text.html': set(['png', 'txt']), | |
| } | |
| self.command.execute(self.options(), [], self.tool) | |
| self.assertEqual(self.tool.executive.calls, [ | |
| [ | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-text.html'], | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-text.html'], | |
| ], | |
| [ | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-text.html'], | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-text.html'], | |
| ], | |
| ]) | |
| def test_rebaseline_expectations_noop(self): | |
| self._zero_out_test_expectations() | |
| oc = OutputCapture() | |
| try: | |
| oc.capture_output() | |
| self.command.execute(self.options(), [], self.tool) | |
| finally: | |
| _, _, logs = oc.restore_output() | |
| self.assertEqual(self.tool.filesystem.written_files, {}) | |
| self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n') | |
| def disabled_test_overrides_are_included_correctly(self): | |
| # TODO(qyearsley): Fix or remove this test method. | |
| # This tests that any tests marked as REBASELINE in the overrides are found, but | |
| # that the overrides do not get written into the main file. | |
| self._zero_out_test_expectations() | |
| self._write(self.mac_expectations_path, '') | |
| self.mac_port.expectations_dict = lambda: { | |
| self.mac_expectations_path: '', | |
| 'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n' | |
| 'Bug(y) userscripts/test.html [ Crash ]\n')} | |
| self._write('/userscripts/another-test.html', '') | |
| self.assertDictEqual(self.command._tests_to_rebaseline(self.mac_port), | |
| {'userscripts/another-test.html': set(['png', 'txt', 'wav'])}) | |
| self.assertEqual(self._read(self.mac_expectations_path), '') | |
| def test_rebaseline_without_other_expectations(self): | |
| self._write('userscripts/another-test.html', 'Dummy test contents') | |
| self._write(self.mac_expectations_path, 'Bug(x) userscripts/another-test.html [ Rebaseline ]\n') | |
| self.assertDictEqual(self.command._tests_to_rebaseline(self.mac_port), | |
| {'userscripts/another-test.html': ('png', 'wav', 'txt')}) | |
| def test_rebaseline_test_passes_everywhere(self): | |
| test_port = self.tool.port_factory.get('test') | |
| for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']: | |
| self.tool.buildbot.set_results(Build(builder), LayoutTestResults({ | |
| 'tests': { | |
| 'fast': { | |
| 'dom': { | |
| 'prototype-taco.html': { | |
| 'expected': 'FAIL', | |
| 'actual': 'PASS', | |
| 'is_unexpected': True | |
| } | |
| } | |
| } | |
| } | |
| })) | |
| self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ | |
| Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ] | |
| """) | |
| self._write_test_file(test_port, 'fast/dom/prototype-taco.html', 'Dummy test contents') | |
| self.tool.executive = MockLineRemovingExecutive() | |
| self.tool.builders = BuilderList({ | |
| 'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']}, | |
| 'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']}, | |
| }) | |
| self.command.execute(self.options(), [], self.tool) | |
| self.assertEqual(self.tool.executive.calls, []) | |
| # The mac ports should both be removed since they're the only ones in the builder list. | |
| self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ | |
| Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ] | |
| """) | |
| def test_rebaseline_missing(self): | |
| self.tool.buildbot.set_results(Build('MOCK Mac10.10'), LayoutTestResults({ | |
| 'tests': { | |
| 'fast': { | |
| 'dom': { | |
| 'missing-text.html': { | |
| 'expected': 'PASS', | |
| 'actual': 'MISSING', | |
| 'is_unexpected': True, | |
| 'is_missing_text': True | |
| }, | |
| 'missing-text-and-image.html': { | |
| 'expected': 'PASS', | |
| 'actual': 'MISSING', | |
| 'is_unexpected': True, | |
| 'is_missing_text': True, | |
| 'is_missing_image': True | |
| }, | |
| 'missing-image.html': { | |
| 'expected': 'PASS', | |
| 'actual': 'MISSING', | |
| 'is_unexpected': True, | |
| 'is_missing_image': True | |
| } | |
| } | |
| } | |
| } | |
| })) | |
| self._write('fast/dom/missing-text.html', 'Dummy test contents') | |
| self._write('fast/dom/missing-text-and-image.html', 'Dummy test contents') | |
| self._write('fast/dom/missing-image.html', 'Dummy test contents') | |
| self.command._tests_to_rebaseline = lambda port: { | |
| 'fast/dom/missing-text.html': set(['txt', 'png']), | |
| 'fast/dom/missing-text-and-image.html': set(['txt', 'png']), | |
| 'fast/dom/missing-image.html': set(['txt', 'png']), | |
| } | |
| self.command.execute(self.options(), [], self.tool) | |
| self.assertEqual(self.tool.executive.calls, [ | |
| [ | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.html'], | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-and-image.html'], | |
| ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', | |
| '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image.html'], | |
| ], | |
| [ | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', | |
| '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.html'], | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', | |
| '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-and-image.html'], | |
| ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', | |
| '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image.html'], | |
| ] | |
| ]) | |
| class MockLineRemovingExecutive(MockExecutive): | |
| def run_in_parallel(self, commands): | |
| assert len(commands) | |
| num_previous_calls = len(self.calls) | |
| command_outputs = [] | |
| for cmd_line, cwd in commands: | |
| out = self.run_command(cmd_line, cwd=cwd) | |
| if 'rebaseline-test-internal' in cmd_line: | |
| out = '{"remove-lines": [{"test": "%s", "builder": "%s"}]}\n' % (cmd_line[8], cmd_line[6]) | |
| command_outputs.append([0, out, '']) | |
| new_calls = self.calls[num_previous_calls:] | |
| self.calls = self.calls[:num_previous_calls] | |
| self.calls.append(new_calls) | |
| return command_outputs |