| # Copyright 2016 The Chromium Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """A command to fetch new baselines from try jobs for the current CL.""" | |
| import json | |
| import logging | |
| import optparse | |
| from webkitpy.common.net.git_cl import GitCL | |
| from webkitpy.layout_tests.models.test_expectations import BASELINE_SUFFIX_LIST | |
| from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand | |
| from webkitpy.w3c.wpt_manifest import WPTManifest | |
| _log = logging.getLogger(__name__) | |
| class RebaselineCL(AbstractParallelRebaselineCommand): | |
| name = 'rebaseline-cl' | |
| help_text = 'Fetches new baselines for a CL from test runs on try bots.' | |
| long_help = ('By default, this command will check the latest try job results ' | |
| 'for all platforms, and start try jobs for platforms with no ' | |
| 'try jobs. Then, new baselines are downloaded for any tests ' | |
| 'that are being rebaselined. After downloading, the baselines ' | |
| 'for different platforms will be optimized (consolidated).') | |
| show_in_main_help = True | |
| def __init__(self): | |
| super(RebaselineCL, self).__init__(options=[ | |
| optparse.make_option( | |
| '--dry-run', action='store_true', default=False, | |
| help='Dry run mode; list actions that would be performed but do not do anything.'), | |
| optparse.make_option( | |
| '--only-changed-tests', action='store_true', default=False, | |
| help='Only download new baselines for tests that are changed in the CL.'), | |
| optparse.make_option( | |
| '--no-trigger-jobs', dest='trigger_jobs', action='store_false', default=True, | |
| help='Do not trigger any try jobs.'), | |
| self.no_optimize_option, | |
| self.results_directory_option, | |
| ]) | |
| def execute(self, options, args, tool): | |
| self._tool = tool | |
| # TODO(qyearsley): Move this call to somewhere else. | |
| WPTManifest.ensure_manifest(tool) | |
| unstaged_baselines = self.unstaged_baselines() | |
| if unstaged_baselines: | |
| _log.error('Aborting: there are unstaged baselines:') | |
| for path in unstaged_baselines: | |
| _log.error(' %s', path) | |
| return 1 | |
| issue_number = self._get_issue_number() | |
| if issue_number is None: | |
| _log.error('No issue number for current branch.') | |
| return 1 | |
| _log.debug('Issue number for current branch: %s', issue_number) | |
| builds = self.git_cl().latest_try_jobs(self._try_bots()) | |
| builders_with_pending_builds = self.builders_with_pending_builds(builds) | |
| if builders_with_pending_builds: | |
| _log.info('There are existing pending builds for:') | |
| for builder in sorted(builders_with_pending_builds): | |
| _log.info(' %s', builder) | |
| builders_with_no_results = self.builders_with_no_results(builds) | |
| if options.trigger_jobs and builders_with_no_results: | |
| self.trigger_builds(builders_with_no_results) | |
| _log.info('Please re-run webkit-patch rebaseline-cl once all pending try jobs have finished.') | |
| return 1 | |
| if builders_with_no_results: | |
| # TODO(qyearsley): Support trying to continue as long as there are | |
| # some results from some builder; see http://crbug.com/673966. | |
| _log.error('The following builders have no results:') | |
| for builder in builders_with_no_results: | |
| _log.error(' %s', builder) | |
| return 1 | |
| _log.debug('Getting results for issue %d.', issue_number) | |
| builds_to_results = self._fetch_results(builds) | |
| if builds_to_results is None: | |
| return 1 | |
| test_prefix_list = {} | |
| if args: | |
| for test in args: | |
| test_prefix_list[test] = {b: BASELINE_SUFFIX_LIST for b in builds} | |
| else: | |
| test_prefix_list = self._test_prefix_list( | |
| builds_to_results, | |
| only_changed_tests=options.only_changed_tests) | |
| self._log_test_prefix_list(test_prefix_list) | |
| if not options.dry_run: | |
| self.rebaseline(options, test_prefix_list) | |
| return 0 | |
| def _get_issue_number(self): | |
| """Returns the current CL issue number, or None.""" | |
| issue = self.git_cl().get_issue_number() | |
| if not issue.isdigit(): | |
| return None | |
| return int(issue) | |
| def git_cl(self): | |
| """Returns a GitCL instance. Can be overridden for tests.""" | |
| return GitCL(self._tool) | |
| def trigger_builds(self, builders): | |
| _log.info('Triggering try jobs for:') | |
| for builder in sorted(builders): | |
| _log.info(' %s', builder) | |
| self.git_cl().trigger_try_jobs(builders) | |
| def builders_with_no_results(self, builds): | |
| """Returns the set of builders that don't have finished results.""" | |
| builders_with_no_builds = set(self._try_bots()) - {b.builder_name for b in builds} | |
| return builders_with_no_builds | self.builders_with_pending_builds(builds) | |
| def builders_with_pending_builds(self, builds): | |
| """Returns the set of builders that have pending builds.""" | |
| return {b.builder_name for b in builds if b.build_number is None} | |
| def _try_bots(self): | |
| """Returns a collection of try bot builders to fetch results for.""" | |
| return self._tool.builders.all_try_builder_names() | |
| def _fetch_results(self, builds): | |
| """Fetches results for all of the given builds. | |
| There should be a one-to-one correspondence between Builds, supported | |
| platforms, and try bots. If not all of the builds can be fetched, then | |
| continuing with rebaselining may yield incorrect results, when the new | |
| baselines are deduped, an old baseline may be kept for the platform | |
| that's missing results. | |
| Args: | |
| builds: A list of Build objects. | |
| Returns: | |
| A dict mapping Build to LayoutTestResults, or None if any results | |
| were not available. | |
| """ | |
| buildbot = self._tool.buildbot | |
| results = {} | |
| for build in builds: | |
| results_url = buildbot.results_url(build.builder_name, build.build_number) | |
| layout_test_results = buildbot.fetch_results(build) | |
| if layout_test_results is None: | |
| _log.error( | |
| 'Failed to fetch results from "%s".\n' | |
| 'Try starting a new job for %s by running :\n' | |
| ' git cl try -b %s', | |
| results_url, build.builder_name, build.builder_name) | |
| return None | |
| results[build] = layout_test_results | |
| return results | |
| def _test_prefix_list(self, builds_to_results, only_changed_tests): | |
| """Returns a dict which lists the set of baselines to fetch. | |
| The dict that is returned is a dict of tests to Build objects | |
| to baseline file extensions. | |
| Args: | |
| builds_to_results: A dict mapping Builds to LayoutTestResults. | |
| only_changed_tests: Whether to only include baselines for tests that | |
| are changed in this CL. If False, all new baselines for failing | |
| tests will be downloaded, even for tests that were not modified. | |
| Returns: | |
| A dict containing information about which new baselines to download. | |
| """ | |
| builds_to_tests = {} | |
| for build, results in builds_to_results.iteritems(): | |
| builds_to_tests[build] = self._tests_to_rebaseline(build, results) | |
| if only_changed_tests: | |
| files_in_cl = self._tool.git().changed_files(diff_filter='AM') | |
| # In the changed files list from Git, paths always use "/" as | |
| # the path separator, and they're always relative to repo root. | |
| # TODO(qyearsley): Do this without using a hard-coded constant. | |
| test_base = 'third_party/WebKit/LayoutTests/' | |
| tests_in_cl = [f[len(test_base):] for f in files_in_cl if f.startswith(test_base)] | |
| result = {} | |
| for build, tests in builds_to_tests.iteritems(): | |
| for test in tests: | |
| if only_changed_tests and test not in tests_in_cl: | |
| continue | |
| if test not in result: | |
| result[test] = {} | |
| result[test][build] = BASELINE_SUFFIX_LIST | |
| return result | |
| def _tests_to_rebaseline(self, build, layout_test_results): | |
| """Fetches a list of tests that should be rebaselined for some build.""" | |
| unexpected_results = layout_test_results.didnt_run_as_expected_results() | |
| tests = sorted(r.test_name() for r in unexpected_results | |
| if r.is_missing_baseline() or r.has_mismatch_result()) | |
| new_failures = self._fetch_tests_with_new_failures(build) | |
| if new_failures is None: | |
| _log.warning('No retry summary available for build %s.', build) | |
| else: | |
| tests = [t for t in tests if t in new_failures] | |
| return tests | |
| def _fetch_tests_with_new_failures(self, build): | |
| """For a given try job, lists tests that only occurred with the patch.""" | |
| buildbot = self._tool.buildbot | |
| content = buildbot.fetch_retry_summary_json(build) | |
| if content is None: | |
| return None | |
| try: | |
| retry_summary = json.loads(content) | |
| return retry_summary['failures'] | |
| except (ValueError, KeyError): | |
| _log.warning('Unexpected retry summary content:\n%s', content) | |
| return None | |
| @staticmethod | |
| def _log_test_prefix_list(test_prefix_list): | |
| """Logs the tests to download new baselines for.""" | |
| if not test_prefix_list: | |
| _log.info('No tests to rebaseline; exiting.') | |
| return | |
| _log.debug('Tests to rebaseline:') | |
| for test, builds in test_prefix_list.iteritems(): | |
| _log.debug(' %s:', test) | |
| for build in sorted(builds): | |
| _log.debug(' %s', build) |