| # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions | |
| # are met: | |
| # | |
| # 1. Redistributions of source code must retain the above | |
| # copyright notice, this list of conditions and the following | |
| # disclaimer. | |
| # 2. Redistributions in binary form must reproduce the above | |
| # copyright notice, this list of conditions and the following | |
| # disclaimer in the documentation and/or other materials | |
| # provided with the distribution. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY | |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
| # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE | |
| # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, | |
| # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | |
| # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR | |
| # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR | |
| # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF | |
| # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
| # SUCH DAMAGE. | |
| """Logic for converting and copying files from a W3C repo. | |
| This module is responsible for modifying and copying a subset of the tests from | |
| a local W3C repository source directory into a destination directory. | |
| """ | |
| import logging | |
| import mimetypes | |
| import re | |
| from webkitpy.common.webkit_finder import WebKitFinder | |
| from webkitpy.layout_tests.models.test_expectations import TestExpectationParser | |
| from webkitpy.w3c.test_parser import TestParser | |
| from webkitpy.w3c.test_converter import convert_for_webkit | |
| _log = logging.getLogger(__name__) | |
| class TestCopier(object): | |
| def __init__(self, host, source_repo_path, dest_dir_name='external'): | |
| """Initializes variables to prepare for copying and converting files. | |
| Args: | |
| host: An instance of Host. | |
| source_repo_path: Path to the local checkout of a | |
| web-platform-tests or csswg-test repository. | |
| dest_dir_name: The name of the directory under the layout tests | |
| directory where imported tests should be copied to. | |
| TODO(qyearsley): This can be made into a constant. | |
| """ | |
| self.host = host | |
| assert self.host.filesystem.exists(source_repo_path) | |
| self.source_repo_path = source_repo_path | |
| self.dest_dir_name = dest_dir_name | |
| self.filesystem = self.host.filesystem | |
| self.webkit_finder = WebKitFinder(self.filesystem) | |
| self._webkit_root = self.webkit_finder.webkit_base() | |
| self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('LayoutTests') | |
| self.destination_directory = self.filesystem.normpath( | |
| self.filesystem.join( | |
| self.layout_tests_dir, | |
| dest_dir_name, | |
| self.filesystem.basename(self.source_repo_path))) | |
| self.import_in_place = (self.source_repo_path == self.destination_directory) | |
| self.dir_above_repo = self.filesystem.dirname(self.source_repo_path) | |
| self.is_wpt = self.filesystem.basename(self.source_repo_path) == 'wpt' | |
| self.import_list = [] | |
| # This is just a FYI list of CSS properties that still need to be prefixed, | |
| # which may be output after importing. | |
| self._prefixed_properties = {} | |
| def do_import(self): | |
| _log.info('Importing %s into %s', self.source_repo_path, self.destination_directory) | |
| self.find_importable_tests() | |
| self.import_tests() | |
| def find_importable_tests(self): | |
| """Walks through the source directory to find what tests should be imported. | |
| This function sets self.import_list, which contains information about how many | |
| tests are being imported, and their source and destination paths. | |
| """ | |
| paths_to_skip = self.find_paths_to_skip() | |
| for root, dirs, files in self.filesystem.walk(self.source_repo_path): | |
| cur_dir = root.replace(self.dir_above_repo + '/', '') + '/' | |
| _log.debug('Scanning %s...', cur_dir) | |
| total_tests = 0 | |
| reftests = 0 | |
| jstests = 0 | |
| # Files in 'tools' are not for browser testing, so we skip them. | |
| # See: http://web-platform-tests.org/writing-tests/general-guidelines.html#tools | |
| dirs_to_skip = ('.git', 'test-plan', 'tools') | |
| # We copy all files in 'support', including HTML without metadata. | |
| # See: http://web-platform-tests.org/writing-tests/general-guidelines.html#support-files | |
| dirs_to_include = ('resources', 'support') | |
| if dirs: | |
| for name in dirs_to_skip: | |
| if name in dirs: | |
| dirs.remove(name) | |
| for path in paths_to_skip: | |
| path_base = path.replace(self.dest_dir_name + '/', '') | |
| path_base = path_base.replace(cur_dir, '') | |
| path_full = self.filesystem.join(root, path_base) | |
| if path_base in dirs: | |
| _log.info('Skipping: %s', path_full) | |
| dirs.remove(path_base) | |
| if self.import_in_place: | |
| self.filesystem.rmtree(path_full) | |
| copy_list = [] | |
| for filename in files: | |
| path_full = self.filesystem.join(root, filename) | |
| path_base = path_full.replace(self.source_repo_path + '/', '') | |
| path_base = self.destination_directory.replace(self.layout_tests_dir + '/', '') + '/' + path_base | |
| if path_base in paths_to_skip: | |
| if self.import_in_place: | |
| _log.debug('Pruning: %s', path_base) | |
| self.filesystem.remove(path_full) | |
| continue | |
| else: | |
| continue | |
| # FIXME: This block should really be a separate function, but the early-continues make that difficult. | |
| # TODO(qyearsley): Remove the below block. | |
| if filename != '.gitignore' and (filename.startswith('.') or filename.endswith('.pl')): | |
| _log.debug('Skipping: %s', path_full) | |
| _log.debug(' Reason: Hidden files and perl scripts are not necessary.') | |
| continue | |
| if filename == 'OWNERS' or filename == 'reftest.list': | |
| # See http://crbug.com/584660 and http://crbug.com/582838. | |
| _log.debug('Skipping: %s', path_full) | |
| _log.debug(' Reason: This file may cause Chromium presubmit to fail.') | |
| continue | |
| mimetype = mimetypes.guess_type(path_full) | |
| if ('html' not in str(mimetype[0]) and | |
| 'application/xhtml+xml' not in str(mimetype[0]) and | |
| 'application/xml' not in str(mimetype[0])): | |
| copy_list.append({'src': path_full, 'dest': filename}) | |
| continue | |
| if self.filesystem.basename(root) in dirs_to_include: | |
| copy_list.append({'src': path_full, 'dest': filename}) | |
| continue | |
| test_parser = TestParser(path_full, self.host) | |
| test_info = test_parser.analyze_test() | |
| if test_info is None: | |
| copy_list.append({'src': path_full, 'dest': filename}) | |
| continue | |
| if 'reference' in test_info.keys(): | |
| ref_path_full = test_info['reference'] | |
| if not self.filesystem.exists(ref_path_full): | |
| _log.warning('Skipping: %s', path_full) | |
| _log.warning(' Reason: Ref file "%s" was not found.', ref_path_full) | |
| continue | |
| if not self.is_wpt: | |
| # For csswg-test, we still need to add a ref file | |
| # using WebKit naming conventions. See crbug.com/268729. | |
| # FIXME: Remove this when csswg-test is merged into wpt. | |
| test_basename = self.filesystem.basename(test_info['test']) | |
| ref_file = self.filesystem.splitext(test_basename)[0] + '-expected' | |
| ref_file += self.filesystem.splitext(ref_path_full)[1] | |
| copy_list.append({ | |
| 'src': test_info['reference'], | |
| 'dest': ref_file, | |
| 'reference_support_info': test_info['reference_support_info'], | |
| }) | |
| reftests += 1 | |
| total_tests += 1 | |
| copy_list.append({'src': test_info['test'], 'dest': filename}) | |
| elif 'jstest' in test_info.keys(): | |
| jstests += 1 | |
| total_tests += 1 | |
| copy_list.append({'src': path_full, 'dest': filename, 'is_jstest': True}) | |
| if copy_list: | |
| # Only add this directory to the list if there's something to import | |
| self.import_list.append({'dirname': root, 'copy_list': copy_list, | |
| 'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests}) | |
| def find_paths_to_skip(self): | |
| paths_to_skip = set() | |
| port = self.host.port_factory.get() | |
| w3c_import_expectations_path = self.webkit_finder.path_from_webkit_base('LayoutTests', 'W3CImportExpectations') | |
| w3c_import_expectations = self.filesystem.read_text_file(w3c_import_expectations_path) | |
| parser = TestExpectationParser(port, all_tests=(), is_lint_mode=False) | |
| expectation_lines = parser.parse(w3c_import_expectations_path, w3c_import_expectations) | |
| for line in expectation_lines: | |
| if 'SKIP' in line.expectations: | |
| if line.specifiers: | |
| _log.warning('W3CImportExpectations:%s should not have any specifiers', line.line_numbers) | |
| continue | |
| paths_to_skip.add(line.name) | |
| return paths_to_skip | |
| def import_tests(self): | |
| """Reads |self.import_list|, and converts and copies files to their destination.""" | |
| total_imported_tests = 0 | |
| total_imported_reftests = 0 | |
| total_imported_jstests = 0 | |
| for dir_to_copy in self.import_list: | |
| total_imported_tests += dir_to_copy['total_tests'] | |
| total_imported_reftests += dir_to_copy['reftests'] | |
| total_imported_jstests += dir_to_copy['jstests'] | |
| if not dir_to_copy['copy_list']: | |
| continue | |
| orig_path = dir_to_copy['dirname'] | |
| relative_dir = self.filesystem.relpath(orig_path, self.source_repo_path) | |
| dest_dir = self.filesystem.join(self.destination_directory, relative_dir) | |
| if not self.filesystem.exists(dest_dir): | |
| self.filesystem.maybe_make_directory(dest_dir) | |
| copied_files = [] | |
| for file_to_copy in dir_to_copy['copy_list']: | |
| copied_file = self.copy_file(file_to_copy, dest_dir) | |
| if copied_file: | |
| copied_files.append(copied_file) | |
| _log.info('') | |
| _log.info('Import complete') | |
| _log.info('') | |
| _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests) | |
| _log.info('Imported %d reftests', total_imported_reftests) | |
| _log.info('Imported %d JS tests', total_imported_jstests) | |
| _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests) | |
| _log.info('') | |
| if self._prefixed_properties: | |
| _log.info('Properties needing prefixes (by count):') | |
| for prefixed_property in sorted(self._prefixed_properties, key=lambda p: self._prefixed_properties[p]): | |
| _log.info(' %s: %s', prefixed_property, self._prefixed_properties[prefixed_property]) | |
| def copy_file(self, file_to_copy, dest_dir): | |
| """Converts and copies a file, if it should be copied. | |
| Args: | |
| file_to_copy: A dict in a file copy list constructed by | |
| find_importable_tests, which represents one file to copy, including | |
| the keys: | |
| "src": Absolute path to the source location of the file. | |
| "destination": File name of the destination file. | |
| And possibly also the keys "reference_support_info" or "is_jstest". | |
| dest_dir: Path to the directory where the file should be copied. | |
| Returns: | |
| The path to the new file, relative to the Blink root (//third_party/WebKit). | |
| """ | |
| source_path = self.filesystem.normpath(file_to_copy['src']) | |
| dest_path = self.filesystem.join(dest_dir, file_to_copy['dest']) | |
| if self.filesystem.isdir(source_path): | |
| _log.error('%s refers to a directory', source_path) | |
| return None | |
| if not self.filesystem.exists(source_path): | |
| _log.error('%s not found. Possible error in the test.', source_path) | |
| return None | |
| reference_support_info = file_to_copy.get('reference_support_info') or None | |
| if not self.filesystem.exists(self.filesystem.dirname(dest_path)): | |
| if not self.import_in_place: | |
| self.filesystem.maybe_make_directory(self.filesystem.dirname(dest_path)) | |
| relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir) | |
| # FIXME: Maybe doing a file diff is in order here for existing files? | |
| # In other words, there's no sense in overwriting identical files, but | |
| # there's no harm in copying the identical thing. | |
| _log.debug(' copying %s', relpath) | |
| if self.should_try_to_convert(file_to_copy, source_path, dest_dir): | |
| converted_file = convert_for_webkit( | |
| dest_dir, filename=source_path, | |
| reference_support_info=reference_support_info, | |
| host=self.host) | |
| for prefixed_property in converted_file[0]: | |
| self._prefixed_properties.setdefault(prefixed_property, 0) | |
| self._prefixed_properties[prefixed_property] += 1 | |
| self.filesystem.write_text_file(dest_path, converted_file[1]) | |
| else: | |
| if not self.import_in_place: | |
| self.filesystem.copyfile(source_path, dest_path) | |
| if self.filesystem.read_binary_file(source_path)[:2] == '#!': | |
| self.filesystem.make_executable(dest_path) | |
| return dest_path.replace(self._webkit_root, '') | |
| @staticmethod | |
| def should_try_to_convert(file_to_copy, source_path, dest_dir): | |
| """Checks whether we should try to modify the file when importing.""" | |
| if file_to_copy.get('is_jstest', False): | |
| return False | |
| # Conversion is not necessary for any tests in wpt now; see http://crbug.com/654081. | |
| # Note, we want to move away from converting files, see http://crbug.com/663773. | |
| if re.search(r'[/\\]external[/\\]wpt[/\\]', dest_dir): | |
| return False | |
| # Only HTML, XHTML and CSS files should be converted. | |
| mimetype, _ = mimetypes.guess_type(source_path) | |
| return mimetype in ('text/html', 'application/xhtml+xml', 'text/css') |