| # Copyright 2016 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import copy |
| |
| from webkitpy.common.host_mock import MockHost |
| from webkitpy.common.net.buildbot import Build |
| from webkitpy.common.net.buildbot_mock import MockBuildBot |
| from webkitpy.common.net.layout_test_results import LayoutTestResult, LayoutTestResults |
| from webkitpy.common.system.log_testing import LoggingTestCase |
| from webkitpy.layout_tests.builder_list import BuilderList |
| from webkitpy.layout_tests.port.factory_mock import MockPortFactory |
| from webkitpy.w3c.wpt_expectations_updater import WPTExpectationsUpdater, MARKER_COMMENT |
| |
| |
| class WPTExpectationsUpdaterTest(LoggingTestCase): |
| |
| def mock_host(self): |
| super(WPTExpectationsUpdaterTest, self).setUp() |
| host = MockHost() |
| host.port_factory = MockPortFactory(host) |
| host.builders = BuilderList({ |
| 'MOCK Try Mac10.10': { |
| 'port_name': 'test-mac-mac10.10', |
| 'specifiers': ['Mac10.10', 'Release'], |
| 'is_try_builder': True, |
| }, |
| 'MOCK Try Mac10.11': { |
| 'port_name': 'test-mac-mac10.11', |
| 'specifiers': ['Mac10.11', 'Release'], |
| 'is_try_builder': True, |
| }, |
| 'MOCK Try Trusty': { |
| 'port_name': 'test-linux-trusty', |
| 'specifiers': ['Trusty', 'Release'], |
| 'is_try_builder': True, |
| }, |
| 'MOCK Try Precise': { |
| 'port_name': 'test-linux-precise', |
| 'specifiers': ['Precise', 'Release'], |
| 'is_try_builder': True, |
| }, |
| 'MOCK Try Win10': { |
| 'port_name': 'test-win-win10', |
| 'specifiers': ['Win10', 'Release'], |
| 'is_try_builder': True, |
| }, |
| 'MOCK Try Win7': { |
| 'port_name': 'test-win-win7', |
| 'specifiers': ['Win7', 'Release'], |
| 'is_try_builder': True, |
| }, |
| }) |
| return host |
| |
| def test_get_failing_results_dict_only_passing_results(self): |
| host = self.mock_host() |
| host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), LayoutTestResults({ |
| 'tests': { |
| 'x': { |
| 'passing-test.html': { |
| 'expected': 'PASS', |
| 'actual': 'PASS', |
| }, |
| }, |
| }, |
| })) |
| updater = WPTExpectationsUpdater(host) |
| self.assertEqual(updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {}) |
| |
| def test_get_failing_results_dict_no_results(self): |
| host = self.mock_host() |
| host.buildbot = MockBuildBot() |
| host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), None) |
| updater = WPTExpectationsUpdater(host) |
| self.assertEqual(updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {}) |
| |
| def test_get_failing_results_dict_some_failing_results(self): |
| host = self.mock_host() |
| host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), LayoutTestResults({ |
| 'tests': { |
| 'x': { |
| 'failing-test.html': { |
| 'expected': 'PASS', |
| 'actual': 'IMAGE', |
| 'is_unexpected': True, |
| }, |
| }, |
| }, |
| })) |
| updater = WPTExpectationsUpdater(host) |
| results_dict = updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)) |
| self.assertEqual( |
| results_dict, |
| { |
| 'x/failing-test.html': { |
| 'test-mac-mac10.10': { |
| 'actual': 'IMAGE', |
| 'expected': 'PASS', |
| 'bug': 'crbug.com/626703', |
| }, |
| }, |
| }) |
| |
| def test_merge_same_valued_keys_all_match(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| self.assertEqual( |
| updater.merge_same_valued_keys({ |
| 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
| 'two': {'expected': 'FAIL', 'actual': 'PASS'}, |
| }), |
| {('two', 'one'): {'expected': 'FAIL', 'actual': 'PASS'}}) |
| |
| def test_merge_same_valued_keys_one_mismatch(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| self.assertEqual( |
| updater.merge_same_valued_keys({ |
| 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
| 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
| 'three': {'expected': 'FAIL', 'actual': 'PASS'}, |
| }), |
| { |
| ('three', 'one'): {'expected': 'FAIL', 'actual': 'PASS'}, |
| 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
| }) |
| |
| def test_get_expectations(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| self.assertEqual( |
| updater.get_expectations({'expected': 'FAIL', 'actual': 'PASS'}), |
| {'Pass'}) |
| self.assertEqual( |
| updater.get_expectations({'expected': 'FAIL', 'actual': 'TIMEOUT'}), |
| {'Timeout'}) |
| self.assertEqual( |
| updater.get_expectations({'expected': 'TIMEOUT', 'actual': 'PASS'}), |
| {'Pass'}) |
| self.assertEqual( |
| updater.get_expectations({'expected': 'PASS', 'actual': 'TIMEOUT CRASH FAIL'}), |
| {'Crash', 'Failure', 'Timeout'}) |
| self.assertEqual( |
| updater.get_expectations({'expected': 'SLOW CRASH FAIL TIMEOUT', 'actual': 'PASS'}), |
| {'Pass'}) |
| |
| def test_create_line_list_old_tests(self): |
| # In this example, there are two failures that are not in w3c tests. |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| results = { |
| 'fake/test/path.html': { |
| 'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'}, |
| 'two': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'}, |
| } |
| } |
| self.assertEqual(updater.create_line_list(results), []) |
| |
| def test_create_line_list_new_tests(self): |
| # In this example, there are three unexpected results. The new |
| # test expectation lines are sorted by test, and then specifier. |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| results = { |
| 'external/fake/test/zzzz.html': { |
| 'test-mac-mac10.10': {'expected': 'PASS', 'actual': 'FAIL', 'bug': 'crbug.com/test'}, |
| }, |
| 'external/fake/test/path.html': { |
| 'test-linux-trusty': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'}, |
| 'test-mac-mac10.11': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test'}, |
| }, |
| } |
| self.assertEqual( |
| updater.create_line_list(results), |
| [ |
| 'crbug.com/test [ Trusty ] external/fake/test/path.html [ Pass ]', |
| 'crbug.com/test [ Mac10.11 ] external/fake/test/path.html [ Timeout ]', |
| 'crbug.com/test [ Mac10.10 ] external/fake/test/zzzz.html [ Failure ]', |
| ]) |
| |
| def test_specifier_part(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| self.assertEqual(updater.specifier_part(['test-mac-mac10.10'], 'x/y.html'), '[ Mac10.10 ]') |
| self.assertEqual(updater.specifier_part([], 'x/y.html'), '') |
| |
| def test_skipped_specifiers_when_test_is_wontfix(self): |
| host = self.mock_host() |
| expectations_path = '/test.checkout/LayoutTests/NeverFixTests' |
| host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] external/wpt/test.html [ WontFix ]\n' |
| host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html'] = '' |
| updater = WPTExpectationsUpdater(host) |
| self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'), ['Trusty']) |
| |
| def test_simplify_specifiers(self): |
| macros = { |
| 'mac': ['Mac10.10', 'mac10.11'], |
| 'win': ['Win7', 'win10'], |
| 'Linux': ['Trusty'], |
| } |
| self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['mac10.10', 'mac10.11'], macros), ['Mac']) |
| self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11', 'Trusty'], macros), ['Linux', 'Mac']) |
| self.assertEqual( |
| WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11', 'Trusty', 'Win7', 'Win10'], macros), []) |
| self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['a', 'b', 'c'], {}), ['A', 'B', 'C']) |
| self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac', 'Win', 'Linux'], macros), []) |
| |
| def test_specifier_part_with_skipped_test(self): |
| host = self.mock_host() |
| expectations_path = '/test.checkout/LayoutTests/NeverFixTests' |
| host.filesystem.files[expectations_path] = 'crbug.com/111 [ Linux Mac10.11 ] external/wpt/test.html [ WontFix ]\n' |
| host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html'] = '' |
| updater = WPTExpectationsUpdater(host) |
| self.assertEqual( |
| updater.specifier_part(['test-mac-mac10.10', 'test-win-win7', 'test-win-win10'], 'external/wpt/test.html'), '') |
| self.assertEqual( |
| updater.specifier_part(['test-win-win7', 'test-win-win10'], 'external/wpt/another.html'), '[ Win ]') |
| |
| def test_merge_dicts_with_conflict_raise_exception(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| # Both dicts here have the key "one", and the value is not equal. |
| with self.assertRaises(ValueError): |
| updater.merge_dicts( |
| { |
| 'external/fake/test/path.html': { |
| 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
| 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
| 'three': {'expected': 'FAIL', 'actual': 'PASS'}, |
| }, |
| }, |
| { |
| 'external/fake/test/path.html': { |
| 'one': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
| } |
| }) |
| |
| def test_merge_dicts_merges_second_dict_into_first(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| one = { |
| 'fake/test/path.html': { |
| 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
| 'two': {'expected': 'FAIL', 'actual': 'PASS'}, |
| } |
| } |
| two = { |
| 'external/fake/test/path.html': { |
| 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
| 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
| 'three': {'expected': 'FAIL', 'actual': 'PASS'}, |
| } |
| } |
| three = { |
| 'external/fake/test/path.html': { |
| 'four': {'expected': 'FAIL', 'actual': 'PASS'}, |
| } |
| } |
| |
| output = updater.merge_dicts(one, three) |
| self.assertEqual(output, one) |
| output = updater.merge_dicts(two, three) |
| self.assertEqual(output, two) |
| |
| def test_generate_results_dict(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| layout_test_list = [ |
| LayoutTestResult( |
| 'test/name.html', { |
| 'expected': 'bar', |
| 'actual': 'foo', |
| 'is_unexpected': True, |
| 'has_stderr': True, |
| }) |
| ] |
| self.assertEqual(updater.generate_results_dict('test-mac-mac10.10', layout_test_list), { |
| 'test/name.html': { |
| 'test-mac-mac10.10': { |
| 'expected': 'bar', |
| 'actual': 'foo', |
| 'bug': 'crbug.com/626703', |
| } |
| } |
| }) |
| |
| def test_write_to_test_expectations_with_marker_comment(self): |
| host = self.mock_host() |
| |
| expectations_path = host.port_factory.get().path_to_generic_test_expectations_file() |
| host.filesystem.files[expectations_path] = MARKER_COMMENT + '\n' |
| updater = WPTExpectationsUpdater(host) |
| line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] |
| updater.write_to_test_expectations(line_list) |
| value = updater.host.filesystem.read_text_file(expectations_path) |
| self.assertMultiLineEqual( |
| value, |
| (MARKER_COMMENT + '\n' |
| 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) |
| |
| def test_write_to_test_expectations_with_no_marker_comment(self): |
| host = self.mock_host() |
| expectations_path = host.port_factory.get().path_to_generic_test_expectations_file() |
| host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n' |
| updater = WPTExpectationsUpdater(host) |
| line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] |
| updater.write_to_test_expectations(line_list) |
| value = host.filesystem.read_text_file(expectations_path) |
| self.assertMultiLineEqual( |
| value, |
| ('crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n' |
| '\n' + MARKER_COMMENT + '\n' |
| 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]')) |
| |
| def test_write_to_test_expectations_skips_existing_lines(self): |
| host = self.mock_host() |
| expectations_path = host.port_factory.get().path_to_generic_test_expectations_file() |
| host.filesystem.files[expectations_path] = 'crbug.com/111 dont/copy/me.html [ Failure ]\n' |
| updater = WPTExpectationsUpdater(host) |
| line_list = [ |
| 'crbug.com/111 dont/copy/me.html [ Failure ]', |
| 'crbug.com/222 do/copy/me.html [ Failure ]' |
| ] |
| updater.write_to_test_expectations(line_list) |
| value = host.filesystem.read_text_file(expectations_path) |
| self.assertEqual( |
| value, |
| ('crbug.com/111 dont/copy/me.html [ Failure ]\n' |
| '\n' + MARKER_COMMENT + '\n' |
| 'crbug.com/222 do/copy/me.html [ Failure ]')) |
| |
| def test_write_to_test_expectations_with_marker_and_no_lines(self): |
| host = self.mock_host() |
| expectations_path = host.port_factory.get().path_to_generic_test_expectations_file() |
| host.filesystem.files[expectations_path] = ( |
| MARKER_COMMENT + '\n' |
| 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n') |
| updater = WPTExpectationsUpdater(host) |
| updater.write_to_test_expectations([]) |
| value = updater.host.filesystem.read_text_file(expectations_path) |
| self.assertMultiLineEqual( |
| value, |
| (MARKER_COMMENT + '\n' |
| 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) |
| |
| def test_is_js_test_true(self): |
| host = self.mock_host() |
| host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/foo/bar.html'] = ( |
| '<script src="/resources/testharness.js"></script>') |
| updater = WPTExpectationsUpdater(host) |
| self.assertTrue(updater.is_js_test('foo/bar.html')) |
| |
| def test_is_js_test_false(self): |
| host = self.mock_host() |
| host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/foo/bar.html'] = ( |
| '<script src="ref-test.html"></script>') |
| updater = WPTExpectationsUpdater(host) |
| self.assertFalse(updater.is_js_test('foo/bar.html')) |
| |
| def test_is_js_test_non_existent_file(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| self.assertFalse(updater.is_js_test('foo/bar.html')) |
| |
| def test_get_test_to_rebaseline_returns_only_tests_with_failures(self): |
| host = self.mock_host() |
| host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = ( |
| '<script src="/resources/testharness.js"></script>') |
| host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/external/other/test/path.html'] = ( |
| '<script src="/resources/testharness.js"></script>') |
| updater = WPTExpectationsUpdater(host) |
| two = { |
| 'external/fake/test/path.html': { |
| 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
| 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
| 'three': {'expected': 'FAIL', 'actual': 'PASS'}, |
| } |
| } |
| tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) |
| # The other test doesn't have an entry in the test results dict, so it is not listed as a test to rebaseline. |
| self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html']) |
| |
| def test_get_test_to_rebaseline_returns_only_js_tests(self): |
| host = self.mock_host() |
| host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = ( |
| 'this file does not look like a testharness JS test.') |
| updater = WPTExpectationsUpdater(host) |
| two = { |
| 'external/fake/test/path.html': { |
| 'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'}, |
| 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test'}, |
| 'three': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'}, |
| } |
| } |
| tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) |
| self.assertEqual(tests_to_rebaseline, []) |
| |
| def test_get_tests_to_rebaseline_returns_updated_dict(self): |
| host = self.mock_host() |
| test_results_dict = { |
| 'external/fake/test/path.html': { |
| 'one': {'expected': 'PASS', 'actual': 'TEXT'}, |
| 'two': {'expected': 'PASS', 'actual': 'TIMEOUT'}, |
| }, |
| } |
| test_results_dict_copy = copy.deepcopy(test_results_dict) |
| host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = ( |
| '<script src="/resources/testharness.js"></script>') |
| updater = WPTExpectationsUpdater(host) |
| tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline( |
| test_results_dict) |
| self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html']) |
| # The record for the builder with a timeout is kept, but not with a text mismatch, |
| # since that should be covered by downloading a new baseline. |
| self.assertEqual(modified_test_results, { |
| 'external/fake/test/path.html': { |
| 'two': {'expected': 'PASS', 'actual': 'TIMEOUT'}, |
| }, |
| }) |
| # The original dict isn't modified. |
| self.assertEqual(test_results_dict, test_results_dict_copy) |
| |
| def test_get_tests_to_rebaseline_also_returns_slow_tests(self): |
| host = self.mock_host() |
| test_results_dict = { |
| 'external/fake/test/path.html': { |
| 'one': {'expected': 'SLOW', 'actual': 'TEXT'}, |
| 'two': {'expected': 'SLOW', 'actual': 'TIMEOUT'}, |
| }, |
| } |
| test_results_dict_copy = copy.deepcopy(test_results_dict) |
| host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = ( |
| '<script src="/resources/testharness.js"></script>') |
| updater = WPTExpectationsUpdater(host) |
| tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline( |
| test_results_dict) |
| self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html']) |
| # The record for the builder with a timeout is kept, but not with a text mismatch, |
| # since that should be covered by downloading a new baseline. |
| self.assertEqual(modified_test_results, { |
| 'external/fake/test/path.html': { |
| 'two': {'expected': 'SLOW', 'actual': 'TIMEOUT'}, |
| }, |
| }) |
| # The original dict isn't modified. |
| self.assertEqual(test_results_dict, test_results_dict_copy) |
| |
| def test_run_no_issue_number(self): |
| # TODO(qyearsley): For testing: Consider making a MockGitCL class |
| # and use that class to set fake return values when using git cl. |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| updater.get_issue_number = lambda: 'None' |
| self.assertEqual(1, updater.run(args=[])) |
| self.assertLog(['ERROR: No issue on current branch.\n']) |
| |
| def test_run_no_try_results(self): |
| updater = WPTExpectationsUpdater(self.mock_host()) |
| updater.get_latest_try_jobs = lambda: [] |
| self.assertEqual(1, updater.run(args=[])) |
| self.assertLog(['ERROR: No try job information was collected.\n']) |
