| # Copyright 2014 The Chromium Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """Fetches a copy of the latest state of a W3C test repository and commits. | |
| If this script is given the argument --auto-update, it will also: | |
| 1. Upload a CL. | |
| 2. Trigger try jobs and wait for them to complete. | |
| 3. Make any changes that are required for new failing tests. | |
| 4. Commit the CL. | |
| If this script is given the argument --auto-update, it will also attempt to | |
| upload a CL, trigger try jobs, and make any changes that are required for | |
| new failing tests before committing. | |
| """ | |
| import argparse | |
| import logging | |
| from webkitpy.common.net.git_cl import GitCL | |
| from webkitpy.common.webkit_finder import WebKitFinder | |
| from webkitpy.common.net.buildbot import current_build_link | |
| from webkitpy.layout_tests.models.test_expectations import TestExpectations, TestExpectationParser | |
| from webkitpy.layout_tests.port.base import Port | |
| from webkitpy.w3c.common import WPT_REPO_URL, CSS_REPO_URL, WPT_DEST_NAME, CSS_DEST_NAME, exportable_commits_since | |
| from webkitpy.w3c.directory_owners_extractor import DirectoryOwnersExtractor | |
| from webkitpy.w3c.local_wpt import LocalWPT | |
| from webkitpy.w3c.test_copier import TestCopier | |
| from webkitpy.w3c.wpt_expectations_updater import WPTExpectationsUpdater | |
| from webkitpy.w3c.wpt_manifest import WPTManifest | |
| # Settings for how often to check try job results and how long to wait. | |
| POLL_DELAY_SECONDS = 2 * 60 | |
| TIMEOUT_SECONDS = 180 * 60 | |
| _log = logging.getLogger(__file__) | |
| class TestImporter(object): | |
| def __init__(self, host): | |
| self.host = host | |
| self.executive = host.executive | |
| self.fs = host.filesystem | |
| self.finder = WebKitFinder(self.fs) | |
| self.verbose = False | |
| self.git_cl = None | |
| def main(self, argv=None): | |
| options = self.parse_args(argv) | |
| self.verbose = options.verbose | |
| log_level = logging.DEBUG if self.verbose else logging.INFO | |
| logging.basicConfig(level=log_level, format='%(message)s') | |
| if not self.checkout_is_okay(options.allow_local_commits): | |
| return 1 | |
| self.git_cl = GitCL(self.host, auth_refresh_token_json=options.auth_refresh_token_json) | |
| _log.debug('Noting the current Chromium commit.') | |
| _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) | |
| chromium_commit = show_ref_output.split()[0] | |
| assert options.target in ('wpt', 'css') | |
| dest_dir_name = WPT_DEST_NAME | |
| repo_url = WPT_REPO_URL | |
| if options.target != 'wpt': | |
| dest_dir_name = CSS_DEST_NAME | |
| repo_url = CSS_REPO_URL | |
| # TODO(qyearsley): Simplify this to use LocalWPT.fetch when csswg-test | |
| # is merged into web-platform-tests. | |
| temp_repo_path = self.path_from_webkit_base(dest_dir_name) | |
| _log.info('Cloning repo: %s', repo_url) | |
| _log.info('Local path: %s', temp_repo_path) | |
| self.run(['git', 'clone', repo_url, temp_repo_path]) | |
| if options.target == 'wpt' and not options.ignore_exportable_commits: | |
| commits = self.exportable_but_not_exported_commits(temp_repo_path) | |
| if commits: | |
| # If there are exportable commits, then there's no more work | |
| # to do for now. This isn't really an error case; we expect | |
| # to hit this case some of the time. | |
| _log.info('There were exportable but not-yet-exported commits:') | |
| for commit in commits: | |
| _log.info(' https://chromium.googlesource.com/chromium/src/+/%s', commit.sha) | |
| _log.info('Aborting import to prevent clobbering these commits.') | |
| self.clean_up_temp_repo(temp_repo_path) | |
| return 0 | |
| import_commit = self.update(dest_dir_name, temp_repo_path, options.revision) | |
| self.clean_up_temp_repo(temp_repo_path) | |
| if options.target == 'wpt': | |
| self._copy_resources() | |
| has_changes = self._has_changes() | |
| if not has_changes: | |
| _log.info('Done: no changes to import.') | |
| return 0 | |
| commit_message = self._commit_message(chromium_commit, import_commit) | |
| self._commit_changes(commit_message) | |
| _log.info('Done: changes imported and committed.') | |
| if options.auto_update: | |
| commit_successful = self.do_auto_update() | |
| if not commit_successful: | |
| return 1 | |
| return 0 | |
| def parse_args(self, argv): | |
| parser = argparse.ArgumentParser() | |
| parser.description = __doc__ | |
| parser.add_argument('-v', '--verbose', action='store_true', | |
| help='log what we are doing') | |
| parser.add_argument('--allow-local-commits', action='store_true', | |
| help='allow script to run even if we have local commits') | |
| parser.add_argument('-r', dest='revision', action='store', | |
| help='Target revision.') | |
| parser.add_argument('target', choices=['css', 'wpt'], | |
| help='Target repository. "css" for csswg-test, "wpt" for web-platform-tests.') | |
| parser.add_argument('--auto-update', action='store_true', | |
| help='uploads CL and initiates commit queue.') | |
| parser.add_argument('--auth-refresh-token-json', | |
| help='authentication refresh token JSON file, ' | |
| 'used for authentication for try jobs, ' | |
| 'generally not necessary on developer machines') | |
| parser.add_argument('--ignore-exportable-commits', action='store_true', | |
| help='Continue even if there are exportable commits that may be overwritten.') | |
| return parser.parse_args(argv) | |
| def checkout_is_okay(self, allow_local_commits): | |
| git_diff_retcode, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_failure=False) | |
| if git_diff_retcode: | |
| _log.warning('Checkout is dirty; aborting.') | |
| return False | |
| local_commits = self.run(['git', 'log', '--oneline', 'origin/master..HEAD'])[1] | |
| if local_commits and not allow_local_commits: | |
| _log.warning('Checkout has local commits; aborting. Use --allow-local-commits to allow this.') | |
| return False | |
| if self.fs.exists(self.path_from_webkit_base(WPT_DEST_NAME)): | |
| _log.warning('WebKit/%s exists; aborting.', WPT_DEST_NAME) | |
| return False | |
| if self.fs.exists(self.path_from_webkit_base(CSS_DEST_NAME)): | |
| _log.warning('WebKit/%s repo exists; aborting.', CSS_DEST_NAME) | |
| return False | |
| return True | |
| def exportable_but_not_exported_commits(self, wpt_path): | |
| """Checks for commits that might be overwritten by importing. | |
| Args: | |
| wpt_path: The path to a local checkout of web-platform-tests. | |
| Returns: | |
| A list of commits in the Chromium repo that are exportable | |
| but not yet exported to the web-platform-tests repo. | |
| """ | |
| local_wpt = LocalWPT(self.host, path=wpt_path) | |
| assert self.host.filesystem.exists(wpt_path) | |
| _, chromium_commit = local_wpt.most_recent_chromium_commit() | |
| return exportable_commits_since(chromium_commit.sha, self.host, local_wpt) | |
| def clean_up_temp_repo(self, temp_repo_path): | |
| _log.info('Deleting temp repo directory %s.', temp_repo_path) | |
| self.rmtree(temp_repo_path) | |
| def _copy_resources(self): | |
| """Copies resources from wpt to LayoutTests/resources. | |
| We copy idlharness.js and testharness.js in wpt to LayoutTests/resources | |
| in order to use them in non-imported tests. | |
| If this method is changed, the lists of files expected to be identical | |
| in LayoutTests/PRESUBMIT.py should also be changed. | |
| """ | |
| resources_to_copy_from_wpt = [ | |
| ('idlharness.js', 'resources'), | |
| ('testharness.js', 'resources'), | |
| ] | |
| for filename, wpt_subdir in resources_to_copy_from_wpt: | |
| source = self.path_from_webkit_base('LayoutTests', 'external', WPT_DEST_NAME, wpt_subdir, filename) | |
| destination = self.path_from_webkit_base('LayoutTests', 'resources', filename) | |
| self.copyfile(source, destination) | |
| self.run(['git', 'add', destination]) | |
| def _generate_manifest(self, dest_path): | |
| """Generates MANIFEST.json for imported tests. | |
| Args: | |
| dest_path: Path to the destination WPT directory. | |
| Runs the (newly-updated) manifest command if it's found, and then | |
| stages the generated MANIFEST.json in the git index, ready to commit. | |
| """ | |
| if 'css' in dest_path: | |
| # Do nothing for csswg-test. | |
| return | |
| _log.info('Generating MANIFEST.json') | |
| WPTManifest.generate_manifest(self.host, dest_path) | |
| manifest_path = self.fs.join(dest_path, 'MANIFEST.json') | |
| assert self.fs.exists(manifest_path) | |
| manifest_base_path = self.fs.normpath( | |
| self.fs.join(dest_path, '..', 'WPT_BASE_MANIFEST.json')) | |
| self.copyfile(manifest_path, manifest_base_path) | |
| self.run(['git', 'add', manifest_base_path]) | |
| def update(self, dest_dir_name, temp_repo_path, revision): | |
| """Updates an imported repository. | |
| Args: | |
| dest_dir_name: The destination directory name. | |
| temp_repo_path: Path to local checkout of W3C test repo. | |
| revision: A W3C test repo commit hash, or None. | |
| Returns: | |
| A string for the commit description "<destination>@<commitish>". | |
| """ | |
| if revision is not None: | |
| _log.info('Checking out %s', revision) | |
| self.run(['git', 'checkout', revision], cwd=temp_repo_path) | |
| self.run(['git', 'submodule', 'update', '--init', '--recursive'], cwd=temp_repo_path) | |
| _log.info('Noting the revision we are importing.') | |
| _, show_ref_output = self.run(['git', 'show-ref', 'origin/master'], cwd=temp_repo_path) | |
| master_commitish = show_ref_output.split()[0] | |
| _log.info('Cleaning out tests from LayoutTests/external/%s.', dest_dir_name) | |
| dest_path = self.path_from_webkit_base('LayoutTests', 'external', dest_dir_name) | |
| is_not_baseline_filter = lambda fs, dirname, basename: not self.is_baseline(basename) | |
| files_to_delete = self.fs.files_under(dest_path, file_filter=is_not_baseline_filter) | |
| for subpath in files_to_delete: | |
| self.remove('LayoutTests', 'external', subpath) | |
| _log.info('Importing the tests.') | |
| test_copier = TestCopier(self.host, temp_repo_path) | |
| test_copier.do_import() | |
| self.run(['git', 'add', '--all', 'LayoutTests/external/%s' % dest_dir_name]) | |
| self._delete_orphaned_baselines(dest_path) | |
| self._generate_manifest(dest_path) | |
| _log.info('Updating TestExpectations for any removed or renamed tests.') | |
| self.update_all_test_expectations_files(self._list_deleted_tests(), self._list_renamed_tests()) | |
| return '%s@%s' % (dest_dir_name, master_commitish) | |
| def _commit_changes(self, commit_message): | |
| _log.info('Committing changes.') | |
| self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) | |
| def _has_changes(self): | |
| return_code, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_failure=False) | |
| return return_code == 1 | |
| def _commit_message(self, chromium_commit, import_commit): | |
| return ('Import %s\n\n' | |
| 'Using wpt-import in Chromium %s.\n\n' | |
| 'NOEXPORT=true' % | |
| (import_commit, chromium_commit)) | |
| def _delete_orphaned_baselines(self, dest_path): | |
| _log.info('Deleting any orphaned baselines.') | |
| is_baseline_filter = lambda fs, dirname, basename: self.is_baseline(basename) | |
| previous_baselines = self.fs.files_under(dest_path, file_filter=is_baseline_filter) | |
| for sub_path in previous_baselines: | |
| full_baseline_path = self.fs.join(dest_path, sub_path) | |
| if not self._has_corresponding_test(full_baseline_path): | |
| self.fs.remove(full_baseline_path) | |
| def _has_corresponding_test(self, full_baseline_path): | |
| base = full_baseline_path.replace('-expected.txt', '') | |
| return any(self.fs.exists(base + ext) for ext in Port.supported_file_extensions) | |
| @staticmethod | |
| def is_baseline(basename): | |
| # TODO(qyearsley): Find a better, centralized place for this. | |
| # Also, the name for this method should be is_text_baseline. | |
| return basename.endswith('-expected.txt') | |
| def run(self, cmd, exit_on_failure=True, cwd=None, stdin=''): | |
| _log.debug('Running command: %s', ' '.join(cmd)) | |
| cwd = cwd or self.finder.webkit_base() | |
| proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self.executive.PIPE, stdin=self.executive.PIPE, cwd=cwd) | |
| out, err = proc.communicate(stdin) | |
| if proc.returncode or self.verbose: | |
| _log.info('# ret> %d', proc.returncode) | |
| if out: | |
| for line in out.splitlines(): | |
| _log.info('# out> %s', line) | |
| if err: | |
| for line in err.splitlines(): | |
| _log.info('# err> %s', line) | |
| if exit_on_failure and proc.returncode: | |
| self.host.exit(proc.returncode) | |
| return proc.returncode, out | |
| def check_run(self, command): | |
| return_code, out = self.run(command) | |
| if return_code: | |
| raise Exception('%s failed with exit code %d.' % ' '.join(command), return_code) | |
| return out | |
| def copyfile(self, source, destination): | |
| _log.debug('cp %s %s', source, destination) | |
| self.fs.copyfile(source, destination) | |
| def remove(self, *comps): | |
| dest = self.path_from_webkit_base(*comps) | |
| _log.debug('rm %s', dest) | |
| self.fs.remove(dest) | |
| def rmtree(self, *comps): | |
| dest = self.path_from_webkit_base(*comps) | |
| _log.debug('rm -fr %s', dest) | |
| self.fs.rmtree(dest) | |
| def path_from_webkit_base(self, *comps): | |
| return self.finder.path_from_webkit_base(*comps) | |
| def do_auto_update(self): | |
| """Attempts to upload a CL, make any required adjustments, and commit. | |
| This function assumes that the imported repo has already been updated, | |
| and that change has been committed. There may be newly-failing tests, | |
| so before being able to commit these new changes, we may need to update | |
| TestExpectations or download new baselines. | |
| Returns: | |
| True if successfully committed, False otherwise. | |
| """ | |
| self._upload_cl() | |
| _log.info('Issue: %s', self.git_cl.run(['issue']).strip()) | |
| # First, try on Blink try bots in order to get any new baselines. | |
| # TODO(qyearsley): Make this faster by triggering all try jobs in | |
| # one invocation. | |
| _log.info('Triggering try jobs.') | |
| self.git_cl.trigger_try_jobs() | |
| try_results = self.git_cl.wait_for_try_jobs( | |
| poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS) | |
| if not try_results: | |
| self.git_cl.run(['set-close']) | |
| return False | |
| if try_results and self.git_cl.has_failing_try_results(try_results): | |
| self.fetch_new_expectations_and_baselines() | |
| # Trigger CQ and wait for CQ try jobs to finish. | |
| self.git_cl.run(['set-commit', '--gerrit']) | |
| try_results = self.git_cl.wait_for_try_jobs( | |
| poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS) | |
| if not try_results: | |
| _log.error('No try job results.') | |
| self.git_cl.run(['set-close']) | |
| return False | |
| # If the CQ passes, then the issue will be closed. | |
| status = self.git_cl.run(['status' '--field', 'status']).strip() | |
| _log.info('CL status: "%s"', status) | |
| if status not in ('lgtm', 'closed'): | |
| _log.error('CQ appears to have failed; aborting.') | |
| self.git_cl.run(['set-close']) | |
| return False | |
| _log.info('Update completed.') | |
| return True | |
| def _upload_cl(self): | |
| _log.info('Uploading change list.') | |
| directory_owners = self.get_directory_owners() | |
| description = self._cl_description(directory_owners) | |
| self.git_cl.run([ | |
| 'upload', | |
| '-f', | |
| '--gerrit', | |
| '-m', | |
| description, | |
| ] + self._cc_part(directory_owners)) | |
| @staticmethod | |
| def _cc_part(directory_owners): | |
| cc_part = [] | |
| for owner_tuple in sorted(directory_owners): | |
| cc_part.extend('--cc=' + owner for owner in owner_tuple) | |
| return cc_part | |
| def get_directory_owners(self): | |
| """Returns a mapping of email addresses to owners of changed tests.""" | |
| _log.info('Gathering directory owners emails to CC.') | |
| changed_files = self.host.git().changed_files() | |
| extractor = DirectoryOwnersExtractor(self.fs) | |
| extractor.read_owner_map() | |
| return extractor.list_owners(changed_files) | |
| def _cl_description(self, directory_owners): | |
| """Returns a CL description string. | |
| Args: | |
| directory_owners: A dict of tuples of owner names to lists of directories. | |
| """ | |
| description = self.check_run(['git', 'log', '-1', '--format=%B']) | |
| build_link = current_build_link(self.host) | |
| if build_link: | |
| description += 'Build: %s\n\n' % build_link | |
| if directory_owners: | |
| description += self._format_directory_owners(directory_owners) + '\n\n' | |
| description += 'TBR=qyearsley@chromium.org\n' | |
| # Move any NOEXPORT tag to the end of the description. | |
| description = description.replace('NOEXPORT=true', '') | |
| description = description.replace('\n\n\n\n', '\n\n') | |
| description += 'NOEXPORT=true' | |
| return description | |
| @staticmethod | |
| def _format_directory_owners(directory_owners): | |
| message_lines = ['Directory owners for changes in this CL:'] | |
| for owner_tuple, directories in sorted(directory_owners.items()): | |
| message_lines.append(', '.join(owner_tuple) + ':') | |
| message_lines.extend(' ' + d for d in directories) | |
| return '\n'.join(message_lines) | |
| def fetch_new_expectations_and_baselines(self): | |
| """Adds new expectations and downloads baselines based on try job results, then commits and uploads the change.""" | |
| _log.info('Adding test expectations lines to LayoutTests/TestExpectations.') | |
| expectation_updater = WPTExpectationsUpdater(self.host) | |
| expectation_updater.run(args=[]) | |
| message = 'Update test expectations and baselines.' | |
| self.check_run(['git', 'commit', '-a', '-m', message]) | |
| self.git_cl.run(['upload', '-t', message, '--gerrit']) | |
| def update_all_test_expectations_files(self, deleted_tests, renamed_tests): | |
| """Updates all test expectations files for tests that have been deleted or renamed.""" | |
| port = self.host.port_factory.get() | |
| for path, file_contents in port.all_expectations_dict().iteritems(): | |
| parser = TestExpectationParser(port, all_tests=None, is_lint_mode=False) | |
| expectation_lines = parser.parse(path, file_contents) | |
| self._update_single_test_expectations_file(path, expectation_lines, deleted_tests, renamed_tests) | |
| def _update_single_test_expectations_file(self, path, expectation_lines, deleted_tests, renamed_tests): | |
| """Updates single test expectations file.""" | |
| # FIXME: This won't work for removed or renamed directories with test expectations | |
| # that are directories rather than individual tests. | |
| new_lines = [] | |
| changed_lines = [] | |
| for expectation_line in expectation_lines: | |
| if expectation_line.name in deleted_tests: | |
| continue | |
| if expectation_line.name in renamed_tests: | |
| expectation_line.name = renamed_tests[expectation_line.name] | |
| # Upon parsing the file, a "path does not exist" warning is expected | |
| # to be there for tests that have been renamed, and if there are warnings, | |
| # then the original string is used. If the warnings are reset, then the | |
| # expectation line is re-serialized when output. | |
| expectation_line.warnings = [] | |
| changed_lines.append(expectation_line) | |
| new_lines.append(expectation_line) | |
| new_file_contents = TestExpectations.list_to_string(new_lines, reconstitute_only_these=changed_lines) | |
| self.host.filesystem.write_text_file(path, new_file_contents) | |
| def _list_deleted_tests(self): | |
| """Returns a list of layout tests that have been deleted.""" | |
| out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-filter=D', '--name-only']) | |
| deleted_tests = [] | |
| for line in out.splitlines(): | |
| test = self.finder.layout_test_name(line) | |
| if test: | |
| deleted_tests.append(test) | |
| return deleted_tests | |
| def _list_renamed_tests(self): | |
| """Returns a dict mapping source to dest name for layout tests that have been renamed.""" | |
| out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-filter=R', '--name-status']) | |
| renamed_tests = {} | |
| for line in out.splitlines(): | |
| _, source_path, dest_path = line.split() | |
| source_test = self.finder.layout_test_name(source_path) | |
| dest_test = self.finder.layout_test_name(dest_path) | |
| if source_test and dest_test: | |
| renamed_tests[source_test] = dest_test | |
| return renamed_tests |