| #!/usr/bin/env python |
| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Client-side script to send a try job to the try server. It communicates to |
| the try server by either writting to a svn/git repository or by directly |
| connecting to the server by HTTP. |
| """ |
| |
| import contextlib |
| import datetime |
| import errno |
| import getpass |
| import itertools |
| import json |
| import logging |
| import optparse |
| import os |
| import posixpath |
| import re |
| import shutil |
| import sys |
| import tempfile |
| import urllib |
| import urllib2 |
| import urlparse |
| |
| import breakpad # pylint: disable=W0611 |
| |
| import fix_encoding |
| import gcl |
| import gclient_utils |
| import gerrit_util |
| import scm |
| import subprocess2 |
| |
| |
| __version__ = '1.2' |
| |
| |
| # Constants |
| HELP_STRING = "Sorry, Tryserver is not available." |
| USAGE = r"""%prog [options] |
| |
| Client-side script to send a try job to the try server. It communicates to |
| the try server by either writting to a svn repository or by directly connecting |
| to the server by HTTP.""" |
| |
| EPILOG = """ |
| Examples: |
| Send a patch directly from rietveld: |
| %(prog)s -R codereview.chromium.org/1337 |
| --email recipient@example.com --root src |
| |
| Try a change against a particular revision: |
| %(prog)s -r 123 |
| |
| Try a change including changes to a sub repository: |
| %(prog)s -s third_party/WebKit |
| |
| A git patch off a web site (git inserts a/ and b/) and fix the base dir: |
| %(prog)s --url http://url/to/patch.diff --patchlevel 1 --root src |
| |
| Use svn to store the try job, specify an alternate email address and use a |
| premade diff file on the local drive: |
| %(prog)s --email user@example.com |
| --svn_repo svn://svn.chromium.org/chrome-try/try --diff foo.diff |
| |
| Running only on a 'mac' slave with revision 123 and clobber first; specify |
| manually the 3 source files to use for the try job: |
| %(prog)s --bot mac --revision 123 --clobber -f src/a.cc -f src/a.h |
| -f include/b.h |
| """ |
| |
| GIT_PATCH_DIR_BASENAME = os.path.join('git-try', 'patches-git') |
| GIT_BRANCH_FILE = 'ref' |
| _GIT_PUSH_ATTEMPTS = 3 |
| |
| def DieWithError(message): |
| print >> sys.stderr, message |
| sys.exit(1) |
| |
| |
| def RunCommand(args, error_ok=False, error_message=None, **kwargs): |
| try: |
| return subprocess2.check_output(args, shell=False, **kwargs) |
| except subprocess2.CalledProcessError, e: |
| if not error_ok: |
| DieWithError( |
| 'Command "%s" failed.\n%s' % ( |
| ' '.join(args), error_message or e.stdout or '')) |
| return e.stdout |
| |
| |
| def RunGit(args, **kwargs): |
| """Returns stdout.""" |
| return RunCommand(['git'] + args, **kwargs) |
| |
| class Error(Exception): |
| """An error during a try job submission. |
| |
| For this error, trychange.py does not display stack trace, only message |
| """ |
| |
| class InvalidScript(Error): |
| def __str__(self): |
| return self.args[0] + '\n' + HELP_STRING |
| |
| |
| class NoTryServerAccess(Error): |
| def __str__(self): |
| return self.args[0] + '\n' + HELP_STRING |
| |
| def Escape(name): |
| """Escapes characters that could interfere with the file system or try job |
| parsing. |
| """ |
| return re.sub(r'[^\w#-]', '_', name) |
| |
| |
| class SCM(object): |
| """Simplistic base class to implement one function: ProcessOptions.""" |
| def __init__(self, options, path, file_list): |
| items = path.split('@') |
| assert len(items) <= 2 |
| self.checkout_root = os.path.abspath(items[0]) |
| items.append(None) |
| self.diff_against = items[1] |
| self.options = options |
| # Lazy-load file list from the SCM unless files were specified in options. |
| self._files = None |
| self._file_tuples = None |
| if file_list: |
| self._files = file_list |
| self._file_tuples = [('M', f) for f in self.files] |
| self.options.files = None |
| self.codereview_settings = None |
| self.codereview_settings_file = 'codereview.settings' |
| self.toplevel_root = None |
| |
| def GetFileNames(self): |
| """Return the list of files in the diff.""" |
| return self.files |
| |
| def GetCodeReviewSetting(self, key): |
| """Returns a value for the given key for this repository. |
| |
| Uses gcl-style settings from the repository. |
| """ |
| if gcl: |
| gcl_setting = gcl.GetCodeReviewSetting(key) |
| if gcl_setting != '': |
| return gcl_setting |
| if self.codereview_settings is None: |
| self.codereview_settings = {} |
| settings_file = self.ReadRootFile(self.codereview_settings_file) |
| if settings_file: |
| for line in settings_file.splitlines(): |
| if not line or line.lstrip().startswith('#'): |
| continue |
| k, v = line.split(":", 1) |
| self.codereview_settings[k.strip()] = v.strip() |
| return self.codereview_settings.get(key, '') |
| |
| def _GclStyleSettings(self): |
| """Set default settings based on the gcl-style settings from the repository. |
| |
| The settings in the self.options object will only be set if no previous |
| value exists (i.e. command line flags to the try command will override the |
| settings in codereview.settings). |
| """ |
| settings = { |
| 'port': self.GetCodeReviewSetting('TRYSERVER_HTTP_PORT'), |
| 'host': self.GetCodeReviewSetting('TRYSERVER_HTTP_HOST'), |
| 'svn_repo': self.GetCodeReviewSetting('TRYSERVER_SVN_URL'), |
| 'gerrit_url': self.GetCodeReviewSetting('TRYSERVER_GERRIT_URL'), |
| 'git_repo': self.GetCodeReviewSetting('TRYSERVER_GIT_URL'), |
| 'project': self.GetCodeReviewSetting('TRYSERVER_PROJECT'), |
| # Primarily for revision=auto |
| 'revision': self.GetCodeReviewSetting('TRYSERVER_REVISION'), |
| 'root': self.GetCodeReviewSetting('TRYSERVER_ROOT'), |
| 'patchlevel': self.GetCodeReviewSetting('TRYSERVER_PATCHLEVEL'), |
| } |
| logging.info('\n'.join(['%s: %s' % (k, v) |
| for (k, v) in settings.iteritems() if v])) |
| for (k, v) in settings.iteritems(): |
| # Avoid overwriting options already set using command line flags. |
| if v and getattr(self.options, k) is None: |
| setattr(self.options, k, v) |
| |
| def AutomagicalSettings(self): |
| """Determines settings based on supported code review and checkout tools. |
| """ |
| # Try to find gclient or repo root first. |
| if not self.options.no_search: |
| self.toplevel_root = gclient_utils.FindGclientRoot(self.checkout_root) |
| if self.toplevel_root: |
| logging.info('Found .gclient at %s' % self.toplevel_root) |
| else: |
| self.toplevel_root = gclient_utils.FindFileUpwards( |
| os.path.join('..', '.repo'), self.checkout_root) |
| if self.toplevel_root: |
| logging.info('Found .repo dir at %s' |
| % os.path.dirname(self.toplevel_root)) |
| |
| # Parse TRYSERVER_* settings from codereview.settings before falling back |
| # on setting self.options.root manually further down. Otherwise |
| # TRYSERVER_ROOT would never be used in codereview.settings. |
| self._GclStyleSettings() |
| |
| if self.toplevel_root and not self.options.root: |
| assert os.path.abspath(self.toplevel_root) == self.toplevel_root |
| self.options.root = gclient_utils.PathDifference(self.toplevel_root, |
| self.checkout_root) |
| else: |
| self._GclStyleSettings() |
| |
| def ReadRootFile(self, filename): |
| cur = self.checkout_root |
| root = self.toplevel_root or self.checkout_root |
| |
| assert cur.startswith(root), (root, cur) |
| while cur.startswith(root): |
| filepath = os.path.join(cur, filename) |
| if os.path.isfile(filepath): |
| logging.info('Found %s at %s' % (filename, cur)) |
| return gclient_utils.FileRead(filepath) |
| cur = os.path.dirname(cur) |
| logging.warning('Didn\'t find %s' % filename) |
| return None |
| |
| def _SetFileTuples(self, file_tuples): |
| excluded = ['!', '?', 'X', ' ', '~'] |
| def Excluded(f): |
| if f[0][0] in excluded: |
| return True |
| for r in self.options.exclude: |
| if re.search(r, f[1]): |
| logging.info('Ignoring "%s"' % f[1]) |
| return True |
| return False |
| |
| self._file_tuples = [f for f in file_tuples if not Excluded(f)] |
| self._files = [f[1] for f in self._file_tuples] |
| |
| def CaptureStatus(self): |
| """Returns the 'svn status' emulated output as an array of (status, file) |
| tuples.""" |
| raise NotImplementedError( |
| "abstract method -- subclass %s must override" % self.__class__) |
| |
| @property |
| def files(self): |
| if self._files is None: |
| self._SetFileTuples(self.CaptureStatus()) |
| return self._files |
| |
| @property |
| def file_tuples(self): |
| if self._file_tuples is None: |
| self._SetFileTuples(self.CaptureStatus()) |
| return self._file_tuples |
| |
| |
| class SVN(SCM): |
| """Gathers the options and diff for a subversion checkout.""" |
| def __init__(self, *args, **kwargs): |
| SCM.__init__(self, *args, **kwargs) |
| self.checkout_root = scm.SVN.GetCheckoutRoot(self.checkout_root) |
| if not self.options.email: |
| # Assumes the svn credential is an email address. |
| self.options.email = scm.SVN.GetEmail(self.checkout_root) |
| logging.info("SVN(%s)" % self.checkout_root) |
| |
| def ReadRootFile(self, filename): |
| data = SCM.ReadRootFile(self, filename) |
| if data: |
| return data |
| |
| # Try to search on the subversion repository for the file. |
| if not gcl: |
| return None |
| data = gcl.GetCachedFile(filename) |
| logging.debug('%s:\n%s' % (filename, data)) |
| return data |
| |
| def CaptureStatus(self): |
| return scm.SVN.CaptureStatus(None, self.checkout_root) |
| |
| def GenerateDiff(self): |
| """Returns a string containing the diff for the given file list. |
| |
| The files in the list should either be absolute paths or relative to the |
| given root. |
| """ |
| return scm.SVN.GenerateDiff(self.files, self.checkout_root, full_move=True, |
| revision=self.diff_against) |
| |
| |
| class GIT(SCM): |
| """Gathers the options and diff for a git checkout.""" |
| def __init__(self, *args, **kwargs): |
| SCM.__init__(self, *args, **kwargs) |
| self.checkout_root = scm.GIT.GetCheckoutRoot(self.checkout_root) |
| if not self.options.name: |
| self.options.name = scm.GIT.GetPatchName(self.checkout_root) |
| if not self.options.email: |
| self.options.email = scm.GIT.GetEmail(self.checkout_root) |
| if not self.diff_against: |
| self.diff_against = scm.GIT.GetUpstreamBranch(self.checkout_root) |
| if not self.diff_against: |
| raise NoTryServerAccess( |
| "Unable to determine default branch to diff against. " |
| "Verify this branch is set up to track another" |
| "(via the --track argument to \"git checkout -b ...\"") |
| logging.info("GIT(%s)" % self.checkout_root) |
| |
| def CaptureStatus(self): |
| return scm.GIT.CaptureStatus( |
| [], |
| self.checkout_root.replace(os.sep, '/'), |
| self.diff_against) |
| |
| def GenerateDiff(self): |
| if RunGit(['diff-index', 'HEAD']): |
| print 'Cannot try with a dirty tree. You must commit locally first.' |
| return None |
| return scm.GIT.GenerateDiff( |
| self.checkout_root, |
| files=self.files, |
| full_move=True, |
| branch=self.diff_against) |
| |
| |
| def _ParseBotList(botlist, testfilter): |
| """Parses bot configurations from a list of strings.""" |
| bots = [] |
| if testfilter: |
| for bot in itertools.chain.from_iterable(botspec.split(',') |
| for botspec in botlist): |
| tests = set() |
| if ':' in bot: |
| if bot.endswith(':compile'): |
| tests |= set(['compile']) |
| else: |
| raise ValueError( |
| 'Can\'t use both --testfilter and --bot builder:test formats ' |
| 'at the same time') |
| |
| bots.append((bot, tests)) |
| else: |
| for botspec in botlist: |
| botname = botspec.split(':')[0] |
| tests = set() |
| if ':' in botspec: |
| tests |= set(filter(None, botspec.split(':')[1].split(','))) |
| bots.append((botname, tests)) |
| return bots |
| |
| |
| def _ApplyTestFilter(testfilter, bot_spec): |
| """Applies testfilter from CLI. |
| |
| Specifying a testfilter strips off any builder-specified tests (except for |
| compile). |
| """ |
| if testfilter: |
| return [(botname, set(testfilter) | (tests & set(['compile']))) |
| for botname, tests in bot_spec] |
| else: |
| return bot_spec |
| |
| |
| def _GenTSBotSpec(checkouts, change, changed_files, options): |
| bot_spec = [] |
| # Get try slaves from PRESUBMIT.py files if not specified. |
| # Even if the diff comes from options.url, use the local checkout for bot |
| # selection. |
| try: |
| import presubmit_support |
| root_presubmit = checkouts[0].ReadRootFile('PRESUBMIT.py') |
| if not change: |
| if not changed_files: |
| changed_files = checkouts[0].file_tuples |
| change = presubmit_support.Change(options.name, |
| '', |
| checkouts[0].checkout_root, |
| changed_files, |
| options.issue, |
| options.patchset, |
| options.email) |
| masters = presubmit_support.DoGetTryMasters( |
| change, |
| checkouts[0].GetFileNames(), |
| checkouts[0].checkout_root, |
| root_presubmit, |
| options.project, |
| options.verbose, |
| sys.stdout) |
| |
| # Compatibility for old checkouts and bots that were on tryserver.chromium. |
| trybots = masters.get('tryserver.chromium', []) |
| |
| # Compatibility for checkouts that are not using tryserver.chromium |
| # but are stuck with git-try or gcl-try. |
| if not trybots and len(masters) == 1: |
| trybots = masters.values()[0] |
| |
| if trybots: |
| old_style = filter(lambda x: isinstance(x, basestring), trybots) |
| new_style = filter(lambda x: isinstance(x, tuple), trybots) |
| |
| # _ParseBotList's testfilter is set to None otherwise it will complain. |
| bot_spec = _ApplyTestFilter(options.testfilter, |
| _ParseBotList(old_style, None)) |
| |
| bot_spec.extend(_ApplyTestFilter(options.testfilter, new_style)) |
| |
| except ImportError: |
| pass |
| |
| return bot_spec |
| |
| |
| def _ParseSendChangeOptions(bot_spec, options): |
| """Parse common options passed to _SendChangeHTTP, _SendChangeSVN and |
| _SendChangeGit. |
| """ |
| values = [ |
| ('user', options.user), |
| ('name', options.name), |
| ] |
| # A list of options to copy. |
| optional_values = ( |
| 'email', |
| 'revision', |
| 'root', |
| 'patchlevel', |
| 'issue', |
| 'patchset', |
| 'target', |
| 'project', |
| ) |
| for option_name in optional_values: |
| value = getattr(options, option_name) |
| if value: |
| values.append((option_name, value)) |
| |
| # Not putting clobber to optional_names |
| # because it used to have lower-case 'true'. |
| if options.clobber: |
| values.append(('clobber', 'true')) |
| |
| for bot, tests in bot_spec: |
| values.append(('bot', ('%s:%s' % (bot, ','.join(tests))))) |
| |
| return values |
| |
| |
| def _SendChangeHTTP(bot_spec, options): |
| """Send a change to the try server using the HTTP protocol.""" |
| if not options.host: |
| raise NoTryServerAccess('Please use the --host option to specify the try ' |
| 'server host to connect to.') |
| if not options.port: |
| raise NoTryServerAccess('Please use the --port option to specify the try ' |
| 'server port to connect to.') |
| |
| values = _ParseSendChangeOptions(bot_spec, options) |
| values.append(('patch', options.diff)) |
| |
| url = 'http://%s:%s/send_try_patch' % (options.host, options.port) |
| |
| logging.info('Sending by HTTP') |
| logging.info(''.join("%s=%s\n" % (k, v) for k, v in values)) |
| logging.info(url) |
| logging.info(options.diff) |
| if options.dry_run: |
| return |
| |
| try: |
| logging.info('Opening connection...') |
| connection = urllib2.urlopen(url, urllib.urlencode(values)) |
| logging.info('Done') |
| except IOError, e: |
| logging.info(str(e)) |
| if bot_spec and len(e.args) > 2 and e.args[2] == 'got a bad status line': |
| raise NoTryServerAccess('%s is unaccessible. Bad --bot argument?' % url) |
| else: |
| raise NoTryServerAccess('%s is unaccessible. Reason: %s' % (url, |
| str(e.args))) |
| if not connection: |
| raise NoTryServerAccess('%s is unaccessible.' % url) |
| logging.info('Reading response...') |
| response = connection.read() |
| logging.info('Done') |
| if response != 'OK': |
| raise NoTryServerAccess('%s is unaccessible. Got:\n%s' % (url, response)) |
| |
| PrintSuccess(bot_spec, options) |
| |
| @contextlib.contextmanager |
| def _TempFilename(name, contents=None): |
| """Create a temporary directory, append the specified name and yield. |
| |
| In contrast to NamedTemporaryFile, does not keep the file open. |
| Deletes the file on __exit__. |
| """ |
| temp_dir = tempfile.mkdtemp(prefix=name) |
| try: |
| path = os.path.join(temp_dir, name) |
| if contents is not None: |
| with open(path, 'wb') as f: |
| f.write(contents) |
| yield path |
| finally: |
| shutil.rmtree(temp_dir, True) |
| |
| |
| @contextlib.contextmanager |
| def _PrepareDescriptionAndPatchFiles(description, options): |
| """Creates temporary files with description and patch. |
| |
| __enter__ called on the return value returns a tuple of patch_filename and |
| description_filename. |
| |
| Args: |
| description: contents of description file. |
| options: patchset options object. Must have attributes: user, |
| name (of patch) and diff (contents of patch). |
| """ |
| current_time = str(datetime.datetime.now()).replace(':', '.') |
| patch_basename = '%s.%s.%s.diff' % (Escape(options.user), |
| Escape(options.name), current_time) |
| with _TempFilename('description', description) as description_filename: |
| with _TempFilename(patch_basename, options.diff) as patch_filename: |
| yield patch_filename, description_filename |
| |
| |
| def _SendChangeSVN(bot_spec, options): |
| """Send a change to the try server by committing a diff file on a subversion |
| server.""" |
| if not options.svn_repo: |
| raise NoTryServerAccess('Please use the --svn_repo option to specify the' |
| ' try server svn repository to connect to.') |
| |
| values = _ParseSendChangeOptions(bot_spec, options) |
| description = ''.join("%s=%s\n" % (k, v) for k, v in values) |
| logging.info('Sending by SVN') |
| logging.info(description) |
| logging.info(options.svn_repo) |
| logging.info(options.diff) |
| if options.dry_run: |
| return |
| |
| with _PrepareDescriptionAndPatchFiles(description, options) as ( |
| patch_filename, description_filename): |
| if sys.platform == "cygwin": |
| # Small chromium-specific issue here: |
| # git-try uses /usr/bin/python on cygwin but svn.bat will be used |
| # instead of /usr/bin/svn by default. That causes bad things(tm) since |
| # Windows' svn.exe has no clue about cygwin paths. Hence force to use |
| # the cygwin version in this particular context. |
| exe = "/usr/bin/svn" |
| else: |
| exe = "svn" |
| patch_dir = os.path.dirname(patch_filename) |
| command = [exe, 'import', '-q', patch_dir, options.svn_repo, '--file', |
| description_filename] |
| if scm.SVN.AssertVersion("1.5")[0]: |
| command.append('--no-ignore') |
| |
| try: |
| subprocess2.check_call(command) |
| except subprocess2.CalledProcessError, e: |
| raise NoTryServerAccess(str(e)) |
| |
| PrintSuccess(bot_spec, options) |
| |
| def _GetPatchGitRepo(git_url): |
| """Gets a path to a Git repo with patches. |
| |
| Stores patches in .git/git-try/patches-git directory, a git repo. If it |
| doesn't exist yet or its origin URL is different, cleans up and clones it. |
| If it existed before, then pulls changes. |
| |
| Does not support SVN repo. |
| |
| Returns a path to the directory with patches. |
| """ |
| git_dir = scm.GIT.GetGitDir(os.getcwd()) |
| patch_dir = os.path.join(git_dir, GIT_PATCH_DIR_BASENAME) |
| |
| logging.info('Looking for git repo for patches') |
| # Is there already a repo with the expected url or should we clone? |
| clone = True |
| if os.path.exists(patch_dir) and scm.GIT.IsInsideWorkTree(patch_dir): |
| existing_url = scm.GIT.Capture( |
| ['config', '--local', 'remote.origin.url'], |
| cwd=patch_dir) |
| clone = existing_url != git_url |
| |
| if clone: |
| if os.path.exists(patch_dir): |
| logging.info('Cleaning up') |
| shutil.rmtree(patch_dir, True) |
| logging.info('Cloning patch repo') |
| scm.GIT.Capture(['clone', git_url, GIT_PATCH_DIR_BASENAME], cwd=git_dir) |
| email = scm.GIT.GetEmail(cwd=os.getcwd()) |
| scm.GIT.Capture(['config', '--local', 'user.email', email], cwd=patch_dir) |
| else: |
| if scm.GIT.IsWorkTreeDirty(patch_dir): |
| logging.info('Work dir is dirty: hard reset!') |
| scm.GIT.Capture(['reset', '--hard'], cwd=patch_dir) |
| logging.info('Updating patch repo') |
| scm.GIT.Capture(['pull', 'origin', 'master'], cwd=patch_dir) |
| |
| return os.path.abspath(patch_dir) |
| |
| |
| def _SendChangeGit(bot_spec, options): |
| """Sends a change to the try server by committing a diff file to a GIT repo. |
| |
| Creates a temp orphan branch, commits patch.diff, creates a ref pointing to |
| that commit, deletes the temp branch, checks master out, adds 'ref' file |
| containing the name of the new ref, pushes master and the ref to the origin. |
| |
| TODO: instead of creating a temp branch, use git-commit-tree. |
| """ |
| |
| if not options.git_repo: |
| raise NoTryServerAccess('Please use the --git_repo option to specify the ' |
| 'try server git repository to connect to.') |
| |
| values = _ParseSendChangeOptions(bot_spec, options) |
| comment_subject = '%s.%s' % (options.user, options.name) |
| comment_body = ''.join("%s=%s\n" % (k, v) for k, v in values) |
| description = '%s\n\n%s' % (comment_subject, comment_body) |
| logging.info('Sending by GIT') |
| logging.info(description) |
| logging.info(options.git_repo) |
| logging.info(options.diff) |
| if options.dry_run: |
| return |
| |
| patch_dir = _GetPatchGitRepo(options.git_repo) |
| def patch_git(*args): |
| return scm.GIT.Capture(list(args), cwd=patch_dir) |
| def add_and_commit(filename, comment_filename): |
| patch_git('add', filename) |
| patch_git('commit', '-F', comment_filename) |
| |
| assert scm.GIT.IsInsideWorkTree(patch_dir) |
| assert not scm.GIT.IsWorkTreeDirty(patch_dir) |
| |
| with _PrepareDescriptionAndPatchFiles(description, options) as ( |
| patch_filename, description_filename): |
| logging.info('Committing patch') |
| |
| temp_branch = 'tmp_patch' |
| target_ref = 'refs/patches/%s/%s' % ( |
| Escape(options.user), |
| os.path.basename(patch_filename).replace(' ','_')) |
| target_filename = os.path.join(patch_dir, 'patch.diff') |
| branch_file = os.path.join(patch_dir, GIT_BRANCH_FILE) |
| |
| patch_git('checkout', 'master') |
| try: |
| # Try deleting an existing temp branch, if any. |
| try: |
| patch_git('branch', '-D', temp_branch) |
| logging.debug('Deleted an existing temp branch.') |
| except subprocess2.CalledProcessError: |
| pass |
| # Create a new branch and put the patch there. |
| patch_git('checkout', '--orphan', temp_branch) |
| patch_git('reset') |
| patch_git('clean', '-f') |
| shutil.copyfile(patch_filename, target_filename) |
| add_and_commit(target_filename, description_filename) |
| assert not scm.GIT.IsWorkTreeDirty(patch_dir) |
| |
| # Create a ref and point it to the commit referenced by temp_branch. |
| patch_git('update-ref', target_ref, temp_branch) |
| |
| # Delete the temp ref. |
| patch_git('checkout', 'master') |
| patch_git('branch', '-D', temp_branch) |
| |
| # Update the branch file in the master. |
| def update_branch(): |
| with open(branch_file, 'w') as f: |
| f.write(target_ref) |
| add_and_commit(branch_file, description_filename) |
| |
| update_branch() |
| |
| # Push master and target_ref to origin. |
| logging.info('Pushing patch') |
| for attempt in xrange(_GIT_PUSH_ATTEMPTS): |
| try: |
| patch_git('push', 'origin', 'master', target_ref) |
| except subprocess2.CalledProcessError as e: |
| is_last = attempt == _GIT_PUSH_ATTEMPTS - 1 |
| if is_last: |
| raise NoTryServerAccess(str(e)) |
| # Fetch, reset, update branch file again. |
| patch_git('fetch', 'origin') |
| patch_git('reset', '--hard', 'origin/master') |
| update_branch() |
| except subprocess2.CalledProcessError, e: |
| # Restore state. |
| patch_git('checkout', 'master') |
| patch_git('reset', '--hard', 'origin/master') |
| raise |
| |
| PrintSuccess(bot_spec, options) |
| |
| def _SendChangeGerrit(bot_spec, options): |
| """Posts a try job to a Gerrit change. |
| |
| Reads Change-Id from the HEAD commit, resolves the current revision, checks |
| that local revision matches the uploaded one, posts a try job in form of a |
| message, sets Tryjob-Request label to 1. |
| |
| Gerrit message format: starts with !tryjob, optionally followed by a tryjob |
| definition in JSON format: |
| buildNames: list of strings specifying build names. |
| build_properties: a dict of build properties. |
| """ |
| |
| logging.info('Sending by Gerrit') |
| if not options.gerrit_url: |
| raise NoTryServerAccess('Please use --gerrit_url option to specify the ' |
| 'Gerrit instance url to connect to') |
| gerrit_host = urlparse.urlparse(options.gerrit_url).hostname |
| logging.debug('Gerrit host: %s' % gerrit_host) |
| |
| def GetChangeId(commmitish): |
| """Finds Change-ID of the HEAD commit.""" |
| CHANGE_ID_RGX = '^Change-Id: (I[a-f0-9]{10,})' |
| comment = scm.GIT.Capture(['log', '-1', commmitish, '--format=%b'], |
| cwd=os.getcwd()) |
| change_id_match = re.search(CHANGE_ID_RGX, comment, re.I | re.M) |
| if not change_id_match: |
| raise Error('Change-Id was not found in the HEAD commit. Make sure you ' |
| 'have a Git hook installed that generates and inserts a ' |
| 'Change-Id into a commit message automatically.') |
| change_id = change_id_match.group(1) |
| return change_id |
| |
| def FormatMessage(): |
| # Build job definition. |
| job_def = {} |
| build_properties = {} |
| if options.testfilter: |
| build_properties['testfilter'] = options.testfilter |
| builderNames = [builder for builder, _ in bot_spec] |
| if builderNames: |
| job_def['builderNames'] = builderNames |
| if build_properties: |
| job_def['build_properties'] = build_properties |
| |
| # Format message. |
| msg = '!tryjob' |
| if job_def: |
| msg = '%s %s' % (msg, json.dumps(job_def, sort_keys=True)) |
| return msg |
| |
| def PostTryjob(message): |
| logging.info('Posting gerrit message: %s' % message) |
| if not options.dry_run: |
| # Post a message and set TryJob=1 label. |
| try: |
| gerrit_util.SetReview(gerrit_host, change_id, msg=message, |
| labels={'Tryjob-Request': 1}) |
| except gerrit_util.GerritError, e: |
| if e.http_status == 400: |
| raise Error(e.message) |
| else: |
| raise |
| |
| head_sha = scm.GIT.Capture(['log', '-1', '--format=%H'], cwd=os.getcwd()) |
| |
| change_id = GetChangeId(head_sha) |
| |
| try: |
| # Check that the uploaded revision matches the local one. |
| changes = gerrit_util.GetChangeCurrentRevision(gerrit_host, change_id) |
| except gerrit_util.GerritAuthenticationError, e: |
| raise NoTryServerAccess(e.message) |
| |
| assert len(changes) <= 1, 'Multiple changes with id %s' % change_id |
| if not changes: |
| raise Error('A change %s was not found on the server. Was it uploaded?' % |
| change_id) |
| logging.debug('Found Gerrit change: %s' % changes[0]) |
| if changes[0]['current_revision'] != head_sha: |
| raise Error('Please upload your latest local changes to Gerrit.') |
| |
| # Post a try job. |
| message = FormatMessage() |
| PostTryjob(message) |
| change_url = urlparse.urljoin(options.gerrit_url, |
| '/#/c/%s' % changes[0]['_number']) |
| print('A tryjob was posted on change %s' % change_url) |
| |
| def PrintSuccess(bot_spec, options): |
| if not options.dry_run: |
| text = 'Patch \'%s\' sent to try server' % options.name |
| if bot_spec: |
| text += ': %s' % ', '.join( |
| '%s:%s' % (b[0], ','.join(b[1])) for b in bot_spec) |
| print(text) |
| |
| |
| def GuessVCS(options, path, file_list): |
| """Helper to guess the version control system. |
| |
| NOTE: Very similar to upload.GuessVCS. Doesn't look for hg since we don't |
| support it yet. |
| |
| This examines the path directory, guesses which SCM we're using, and |
| returns an instance of the appropriate class. Exit with an error if we can't |
| figure it out. |
| |
| Returns: |
| A SCM instance. Exits if the SCM can't be guessed. |
| """ |
| __pychecker__ = 'no-returnvalues' |
| real_path = path.split('@')[0] |
| logging.info("GuessVCS(%s)" % path) |
| # Subversion has a .svn in all working directories. |
| if os.path.isdir(os.path.join(real_path, '.svn')): |
| return SVN(options, path, file_list) |
| |
| # Git has a command to test if you're in a git tree. |
| # Try running it, but don't die if we don't have git installed. |
| try: |
| subprocess2.check_output( |
| ['git', 'rev-parse', '--is-inside-work-tree'], cwd=real_path, |
| stderr=subprocess2.VOID) |
| return GIT(options, path, file_list) |
| except OSError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| except subprocess2.CalledProcessError, e: |
| if e.returncode != errno.ENOENT and e.returncode != 128: |
| # ENOENT == 2 = they don't have git installed. |
| # 128 = git error code when not in a repo. |
| logging.warning('Unexpected error code: %s' % e.returncode) |
| raise |
| raise NoTryServerAccess( |
| ( 'Could not guess version control system for %s.\n' |
| 'Are you in a working copy directory?') % path) |
| |
| |
| def GetMungedDiff(path_diff, diff): |
| # Munge paths to match svn. |
| changed_files = [] |
| for i in range(len(diff)): |
| if diff[i].startswith('--- ') or diff[i].startswith('+++ '): |
| new_file = posixpath.join(path_diff, diff[i][4:]).replace('\\', '/') |
| if diff[i].startswith('--- '): |
| file_path = new_file.split('\t')[0].strip() |
| if file_path.startswith('a/'): |
| file_path = file_path[2:] |
| changed_files.append(('M', file_path)) |
| diff[i] = diff[i][0:4] + new_file |
| return (diff, changed_files) |
| |
| |
| class OptionParser(optparse.OptionParser): |
| def format_epilog(self, _): |
| """Removes epilog formatting.""" |
| return self.epilog or '' |
| |
| |
| def gen_parser(prog): |
| # Parse argv |
| parser = OptionParser(usage=USAGE, version=__version__, prog=prog) |
| parser.add_option("-v", "--verbose", action="count", default=0, |
| help="Prints debugging infos") |
| group = optparse.OptionGroup(parser, "Result and status") |
| group.add_option("-u", "--user", default=getpass.getuser(), |
| help="Owner user name [default: %default]") |
| group.add_option("-e", "--email", |
| default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS', |
| os.environ.get('EMAIL_ADDRESS')), |
| help="Email address where to send the results. Use either " |
| "the TRYBOT_RESULTS_EMAIL_ADDRESS environment " |
| "variable or EMAIL_ADDRESS to set the email address " |
| "the try bots report results to [default: %default]") |
| group.add_option("-n", "--name", |
| help="Descriptive name of the try job") |
| group.add_option("--issue", type='int', |
| help="Update rietveld issue try job status") |
| group.add_option("--patchset", type='int', |
| help="Update rietveld issue try job status. This is " |
| "optional if --issue is used, In that case, the " |
| "latest patchset will be used.") |
| group.add_option("--dry_run", action='store_true', |
| help="Don't send the try job. This implies --verbose, so " |
| "it will print the diff.") |
| parser.add_option_group(group) |
| |
| group = optparse.OptionGroup(parser, "Try job options") |
| group.add_option( |
| "-b", "--bot", action="append", |
| help=("IMPORTANT: specify ONE builder per --bot flag. Use it multiple " |
| "times to specify multiple builders. ex: " |
| "'-bwin_rel:ui_tests,webkit_unit_tests -bwin_layout'. See " |
| "the try server waterfall for the builders name and the tests " |
| "available. Can also be used to specify gtest_filter, e.g. " |
| "-bwin_rel:base_unittests:ValuesTest.*Value")) |
| group.add_option("-B", "--print_bots", action="store_true", |
| help="Print bots we would use (e.g. from PRESUBMIT.py)" |
| " and exit. Do not send patch. Like --dry_run" |
| " but less verbose.") |
| group.add_option("-r", "--revision", |
| help="Revision to use for the try job. If 'auto' is " |
| "specified, it is resolved to the revision a patch is " |
| "generated against (Git only). Default: the " |
| "revision will be determined by the try server; see " |
| "its waterfall for more info") |
| group.add_option("-c", "--clobber", action="store_true", |
| help="Force a clobber before building; e.g. don't do an " |
| "incremental build") |
| # TODO(maruel): help="Select a specific configuration, usually 'debug' or " |
| # "'release'" |
| group.add_option("--target", help=optparse.SUPPRESS_HELP) |
| |
| group.add_option("--project", |
| help="Override which project to use. Projects are defined " |
| "server-side to define what default bot set to use") |
| |
| group.add_option( |
| "-t", "--testfilter", action="append", default=[], |
| help=("Apply a testfilter to all the selected builders. Unless the " |
| "builders configurations are similar, use multiple " |
| "--bot <builder>:<test> arguments.")) |
| |
| parser.add_option_group(group) |
| |
| group = optparse.OptionGroup(parser, "Patch to run") |
| group.add_option("-f", "--file", default=[], dest="files", |
| metavar="FILE", action="append", |
| help="Use many times to list the files to include in the " |
| "try, relative to the repository root") |
| group.add_option("--diff", |
| help="File containing the diff to try") |
| group.add_option("--url", |
| help="Url where to grab a patch, e.g. " |
| "http://example.com/x.diff") |
| group.add_option("-R", "--rietveld_url", default="codereview.appspot.com", |
| metavar="URL", |
| help="Has 2 usages, both refer to the rietveld instance: " |
| "Specify which code review patch to use as the try job " |
| "or rietveld instance to update the try job results " |
| "Default:%default") |
| group.add_option("--root", |
| help="Root to use for the patch; base subdirectory for " |
| "patch created in a subdirectory") |
| group.add_option("-p", "--patchlevel", type='int', metavar="LEVEL", |
| help="Used as -pN parameter to patch") |
| group.add_option("-s", "--sub_rep", action="append", default=[], |
| help="Subcheckout to use in addition. This is mainly " |
| "useful for gclient-style checkouts. In git, checkout " |
| "the branch with changes first. Use @rev or " |
| "@branch to specify the " |
| "revision/branch to diff against. If no @branch is " |
| "given the diff will be against the upstream branch. " |
| "If @branch then the diff is branch..HEAD. " |
| "All edits must be checked in.") |
| group.add_option("--no_search", action="store_true", |
| help=("Disable automatic search for gclient or repo " |
| "checkout root.")) |
| group.add_option("-E", "--exclude", action="append", |
| default=['ChangeLog'], metavar='REGEXP', |
| help="Regexp patterns to exclude files. Default: %default") |
| group.add_option("--upstream_branch", action="store", |
| help="Specify the upstream branch to diff against in the " |
| "main checkout") |
| parser.add_option_group(group) |
| |
| group = optparse.OptionGroup(parser, "Access the try server by HTTP") |
| group.add_option("--use_http", |
| action="store_const", |
| const=_SendChangeHTTP, |
| dest="send_patch", |
| help="Use HTTP to talk to the try server [default]") |
| group.add_option("-H", "--host", |
| help="Host address") |
| group.add_option("-P", "--port", type="int", |
| help="HTTP port") |
| parser.add_option_group(group) |
| |
| group = optparse.OptionGroup(parser, "Access the try server with SVN") |
| group.add_option("--use_svn", |
| action="store_const", |
| const=_SendChangeSVN, |
| dest="send_patch", |
| help="Use SVN to talk to the try server") |
| group.add_option("-S", "--svn_repo", |
| metavar="SVN_URL", |
| help="SVN url to use to write the changes in; --use_svn is " |
| "implied when using --svn_repo") |
| parser.add_option_group(group) |
| |
| group = optparse.OptionGroup(parser, "Access the try server with Git") |
| group.add_option("--use_git", |
| action="store_const", |
| const=_SendChangeGit, |
| dest="send_patch", |
| help="Use GIT to talk to the try server") |
| group.add_option("-G", "--git_repo", |
| metavar="GIT_URL", |
| help="GIT url to use to write the changes in; --use_git is " |
| "implied when using --git_repo") |
| parser.add_option_group(group) |
| |
| group = optparse.OptionGroup(parser, "Access the try server with Gerrit") |
| group.add_option("--use_gerrit", |
| action="store_const", |
| const=_SendChangeGerrit, |
| dest="send_patch", |
| help="Use Gerrit to talk to the try server") |
| group.add_option("--gerrit_url", |
| metavar="GERRIT_URL", |
| help="Gerrit url to post a tryjob to; --use_gerrit is " |
| "implied when using --gerrit_url") |
| parser.add_option_group(group) |
| |
| return parser |
| |
| |
| def TryChange(argv, |
| change, |
| swallow_exception, |
| prog=None, |
| extra_epilog=None): |
| """ |
| Args: |
| argv: Arguments and options. |
| change: Change instance corresponding to the CL. |
| swallow_exception: Whether we raise or swallow exceptions. |
| """ |
| parser = gen_parser(prog) |
| epilog = EPILOG % { 'prog': prog } |
| if extra_epilog: |
| epilog += extra_epilog |
| parser.epilog = epilog |
| |
| options, args = parser.parse_args(argv) |
| |
| # If they've asked for help, give it to them |
| if len(args) == 1 and args[0] == 'help': |
| parser.print_help() |
| return 0 |
| |
| # If they've said something confusing, don't spawn a try job until you |
| # understand what they want. |
| if args: |
| parser.error('Extra argument(s) "%s" not understood' % ' '.join(args)) |
| |
| if options.dry_run: |
| options.verbose += 1 |
| |
| LOG_FORMAT = '%(levelname)s %(filename)s(%(lineno)d): %(message)s' |
| if not swallow_exception: |
| if options.verbose == 0: |
| logging.basicConfig(level=logging.WARNING, format=LOG_FORMAT) |
| elif options.verbose == 1: |
| logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) |
| elif options.verbose > 1: |
| logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) |
| |
| logging.debug(argv) |
| |
| if (options.patchlevel is not None and |
| (options.patchlevel < 0 or options.patchlevel > 10)): |
| parser.error( |
| 'Have you tried --port instead? You probably confused -p and -P.') |
| |
| # Strip off any @ in the user, otherwise svn gets confused. |
| options.user = options.user.split('@', 1)[0] |
| |
| if options.rietveld_url: |
| # Try to extract the review number if possible and fix the protocol. |
| if not '://' in options.rietveld_url: |
| options.rietveld_url = 'http://' + options.rietveld_url |
| match = re.match(r'^(.*)/(\d+)/?$', options.rietveld_url) |
| if match: |
| if options.issue or options.patchset: |
| parser.error('Cannot use both --issue and use a review number url') |
| options.issue = int(match.group(2)) |
| options.rietveld_url = match.group(1) |
| |
| try: |
| changed_files = None |
| # Always include os.getcwd() in the checkout settings. |
| path = os.getcwd() |
| |
| file_list = [] |
| if options.files: |
| file_list = options.files |
| elif change: |
| file_list = [f.LocalPath() for f in change.AffectedFiles()] |
| |
| if options.upstream_branch: |
| path += '@' + options.upstream_branch |
| # Clear file list so that the correct list will be retrieved from the |
| # upstream branch. |
| file_list = [] |
| |
| current_vcs = GuessVCS(options, path, file_list) |
| current_vcs.AutomagicalSettings() |
| options = current_vcs.options |
| vcs_is_git = type(current_vcs) is GIT |
| |
| # So far, git_repo doesn't work with SVN |
| if options.git_repo and not vcs_is_git: |
| parser.error('--git_repo option is supported only for GIT repositories') |
| |
| # If revision==auto, resolve it |
| if options.revision and options.revision.lower() == 'auto': |
| if not vcs_is_git: |
| parser.error('--revision=auto is supported only for GIT repositories') |
| options.revision = scm.GIT.Capture( |
| ['rev-parse', current_vcs.diff_against], |
| cwd=path) |
| |
| checkouts = [current_vcs] |
| for item in options.sub_rep: |
| # Pass file_list=None because we don't know the sub repo's file list. |
| checkout = GuessVCS(options, |
| os.path.join(current_vcs.checkout_root, item), |
| None) |
| if checkout.checkout_root in [c.checkout_root for c in checkouts]: |
| parser.error('Specified the root %s two times.' % |
| checkout.checkout_root) |
| checkouts.append(checkout) |
| |
| can_http = options.port and options.host |
| can_svn = options.svn_repo |
| can_git = options.git_repo |
| can_gerrit = options.gerrit_url |
| can_something = can_http or can_svn or can_git or can_gerrit |
| # If there was no transport selected yet, now we must have enough data to |
| # select one. |
| if not options.send_patch and not can_something: |
| parser.error('Please specify an access method.') |
| |
| # Convert options.diff into the content of the diff. |
| if options.url: |
| if options.files: |
| parser.error('You cannot specify files and --url at the same time.') |
| options.diff = urllib2.urlopen(options.url).read() |
| elif options.diff: |
| if options.files: |
| parser.error('You cannot specify files and --diff at the same time.') |
| options.diff = gclient_utils.FileRead(options.diff, 'rb') |
| elif options.issue and options.patchset is None: |
| # Retrieve the patch from rietveld when the diff is not specified. |
| # When patchset is specified, it's because it's done by gcl/git-try. |
| api_url = '%s/api/%d' % (options.rietveld_url, options.issue) |
| logging.debug(api_url) |
| contents = json.loads(urllib2.urlopen(api_url).read()) |
| options.patchset = contents['patchsets'][-1] |
| diff_url = ('%s/download/issue%d_%d.diff' % |
| (options.rietveld_url, options.issue, options.patchset)) |
| diff = GetMungedDiff('', urllib2.urlopen(diff_url).readlines()) |
| options.diff = ''.join(diff[0]) |
| changed_files = diff[1] |
| else: |
| # Use this as the base. |
| root = checkouts[0].checkout_root |
| diffs = [] |
| for checkout in checkouts: |
| raw_diff = checkout.GenerateDiff() |
| if not raw_diff: |
| continue |
| diff = raw_diff.splitlines(True) |
| path_diff = gclient_utils.PathDifference(root, checkout.checkout_root) |
| # Munge it. |
| diffs.extend(GetMungedDiff(path_diff, diff)[0]) |
| if not diffs: |
| logging.error('Empty or non-existant diff, exiting.') |
| return 1 |
| options.diff = ''.join(diffs) |
| |
| if not options.name: |
| if options.issue: |
| options.name = 'Issue %s' % options.issue |
| else: |
| options.name = 'Unnamed' |
| print('Note: use --name NAME to change the try job name.') |
| |
| if not options.email: |
| parser.error('Using an anonymous checkout. Please use --email or set ' |
| 'the TRYBOT_RESULTS_EMAIL_ADDRESS environment variable.') |
| print('Results will be emailed to: ' + options.email) |
| |
| if options.bot: |
| bot_spec = _ApplyTestFilter( |
| options.testfilter, _ParseBotList(options.bot, options.testfilter)) |
| else: |
| bot_spec = _GenTSBotSpec(checkouts, change, changed_files, options) |
| |
| if options.testfilter: |
| bot_spec = _ApplyTestFilter(options.testfilter, bot_spec) |
| |
| if any('triggered' in b[0] for b in bot_spec): |
| print >> sys.stderr, ( |
| 'ERROR You are trying to send a job to a triggered bot. This type of' |
| ' bot requires an\ninitial job from a parent (usually a builder). ' |
| 'Instead send your job to the parent.\nBot list: %s' % bot_spec) |
| return 1 |
| |
| if options.print_bots: |
| print 'Bots which would be used:' |
| for bot in bot_spec: |
| if bot[1]: |
| print ' %s:%s' % (bot[0], ','.join(bot[1])) |
| else: |
| print ' %s' % (bot[0]) |
| return 0 |
| |
| # Determine sending protocol |
| if options.send_patch: |
| # If forced. |
| senders = [options.send_patch] |
| else: |
| # Try sending patch using avaialble protocols |
| all_senders = [ |
| (_SendChangeHTTP, can_http), |
| (_SendChangeSVN, can_svn), |
| (_SendChangeGerrit, can_gerrit), |
| (_SendChangeGit, can_git), |
| ] |
| senders = [sender for sender, can in all_senders if can] |
| |
| # Send the patch. |
| for sender in senders: |
| try: |
| sender(bot_spec, options) |
| return 0 |
| except NoTryServerAccess: |
| is_last = sender == senders[-1] |
| if is_last: |
| raise |
| assert False, "Unreachable code" |
| except Error, e: |
| if swallow_exception: |
| return 1 |
| print >> sys.stderr, e |
| return 1 |
| except (gclient_utils.Error, subprocess2.CalledProcessError), e: |
| print >> sys.stderr, e |
| return 1 |
| return 0 |
| |
| |
| if __name__ == "__main__": |
| fix_encoding.fix_encoding() |
| sys.exit(TryChange(None, None, False)) |
