| # Copyright (c) 2009, Google Inc. All rights reserved. | |
| # Copyright (c) 2009 Apple Inc. All rights reserved. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions are | |
| # met: | |
| # | |
| # * Redistributions of source code must retain the above copyright | |
| # notice, this list of conditions and the following disclaimer. | |
| # * Redistributions in binary form must reproduce the above | |
| # copyright notice, this list of conditions and the following disclaimer | |
| # in the documentation and/or other materials provided with the | |
| # distribution. | |
| # * Neither the name of Google Inc. nor the names of its | |
| # contributors may be used to endorse or promote products derived from | |
| # this software without specific prior written permission. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| import csv | |
| import errno | |
| import logging | |
| import multiprocessing | |
| import os | |
| import signal | |
| import subprocess | |
| import sys | |
| import threading | |
| import time | |
| from webkitpy.common.system.filesystem import FileSystem | |
| _log = logging.getLogger(__name__) | |
| class ScriptError(Exception): | |
| def __init__(self, | |
| message=None, | |
| script_args=None, | |
| exit_code=None, | |
| output=None, | |
| cwd=None, | |
| output_limit=500): | |
| shortened_output = output | |
| if output and output_limit and len(output) > output_limit: | |
| shortened_output = 'Last %s characters of output:\n%s' % (output_limit, output[-output_limit:]) | |
| if not message: | |
| message = 'Failed to run "%s"' % repr(script_args) | |
| if exit_code: | |
| message += ' exit_code: %d' % exit_code | |
| if cwd: | |
| message += ' cwd: %s' % cwd | |
| if shortened_output: | |
| message += '\n\noutput: %s' % shortened_output | |
| Exception.__init__(self, message) | |
| self.script_args = script_args # 'args' is already used by Exception | |
| self.exit_code = exit_code | |
| self.output = output | |
| self.cwd = cwd | |
| def message_with_output(self): | |
| return unicode(self) | |
| def command_name(self): | |
| command_path = self.script_args | |
| if isinstance(command_path, list): | |
| command_path = command_path[0] | |
| return os.path.basename(command_path) | |
| class Executive(object): | |
| PIPE = subprocess.PIPE | |
| STDOUT = subprocess.STDOUT | |
| DEVNULL = open(os.devnull, 'wb') | |
| def _should_close_fds(self): | |
| # We need to pass close_fds=True to work around Python bug #2320 | |
| # (otherwise we can hang when we kill DumpRenderTree when we are running | |
| # multiple threads). See http://bugs.python.org/issue2320 . | |
| # Note that close_fds isn't supported on Windows, but this bug only | |
| # shows up on Mac and Linux. | |
| return sys.platform not in ('win32', 'cygwin') | |
| def cpu_count(self): | |
| return multiprocessing.cpu_count() | |
| def kill_process(self, pid): | |
| """Attempts to kill the given pid. | |
| Will fail silently if pid does not exist or insufficient permissions. | |
| """ | |
| if sys.platform == 'win32': | |
| # We only use taskkill.exe on windows (not cygwin) because subprocess.pid | |
| # is a CYGWIN pid and taskkill.exe expects a windows pid. | |
| # Thankfully os.kill on CYGWIN handles either pid type. | |
| command = ['taskkill.exe', '/f', '/t', '/pid', pid] | |
| # taskkill will exit 128 if the process is not found. We should log. | |
| self.run_command(command, error_handler=self.ignore_error) | |
| return | |
| # According to http://docs.python.org/library/os.html | |
| # os.kill isn't available on Windows. python 2.5.5 os.kill appears | |
| # to work in cygwin, however it occasionally raises EAGAIN. | |
| retries_left = 10 if sys.platform == 'cygwin' else 1 | |
| while retries_left > 0: | |
| try: | |
| retries_left -= 1 | |
| os.kill(pid, signal.SIGKILL) | |
| _ = os.waitpid(pid, os.WNOHANG) | |
| except OSError as error: | |
| if error.errno == errno.EAGAIN: | |
| if retries_left <= 0: | |
| _log.warning('Failed to kill pid %s. Too many EAGAIN errors.', pid) | |
| continue | |
| if error.errno == errno.ESRCH: # The process does not exist. | |
| return | |
| if error.errno == errno.EPIPE: # The process has exited already on cygwin | |
| return | |
| if error.errno == errno.ECHILD: | |
| # Can't wait on a non-child process, but the kill worked. | |
| return | |
| if error.errno == errno.EACCES and sys.platform == 'cygwin': | |
| # Cygwin python sometimes can't kill native processes. | |
| return | |
| raise | |
| def _win32_check_running_pid(self, pid): | |
| # importing ctypes at the top-level seems to cause weird crashes at | |
| # exit under cygwin on apple's win port. Only win32 needs cygwin, so | |
| # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682 | |
| import ctypes | |
| class PROCESSENTRY32(ctypes.Structure): | |
| _fields_ = [('dwSize', ctypes.c_ulong), | |
| ('cntUsage', ctypes.c_ulong), | |
| ('th32ProcessID', ctypes.c_ulong), | |
| ('th32DefaultHeapID', ctypes.POINTER(ctypes.c_ulong)), | |
| ('th32ModuleID', ctypes.c_ulong), | |
| ('cntThreads', ctypes.c_ulong), | |
| ('th32ParentProcessID', ctypes.c_ulong), | |
| ('pcPriClassBase', ctypes.c_ulong), | |
| ('dwFlags', ctypes.c_ulong), | |
| ('szExeFile', ctypes.c_char * 260)] | |
| CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot | |
| Process32First = ctypes.windll.kernel32.Process32First | |
| Process32Next = ctypes.windll.kernel32.Process32Next | |
| CloseHandle = ctypes.windll.kernel32.CloseHandle | |
| TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number | |
| hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) | |
| pe32 = PROCESSENTRY32() | |
| pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) | |
| result = False | |
| if not Process32First(hProcessSnap, ctypes.byref(pe32)): | |
| _log.debug('Failed getting first process.') | |
| CloseHandle(hProcessSnap) | |
| return result | |
| while True: | |
| if pe32.th32ProcessID == pid: | |
| result = True | |
| break | |
| if not Process32Next(hProcessSnap, ctypes.byref(pe32)): | |
| break | |
| CloseHandle(hProcessSnap) | |
| return result | |
| def check_running_pid(self, pid): | |
| """Return True if pid is alive, otherwise return False.""" | |
| _log.debug('Checking whether pid %d is alive.', pid) | |
| if sys.platform == 'win32': | |
| return self._win32_check_running_pid(pid) | |
| try: | |
| os.kill(pid, 0) | |
| return True | |
| except OSError: | |
| return False | |
| def _running_processes(self): | |
| processes = [] | |
| if sys.platform in ('win32', 'cygwin'): | |
| tasklist_process = self.popen(['tasklist', '/fo', 'csv'], | |
| stdout=self.PIPE, stderr=self.PIPE) | |
| stdout, _ = tasklist_process.communicate() | |
| stdout_reader = csv.reader(stdout.splitlines()) | |
| for line in stdout_reader: | |
| processes.append([column for column in line]) | |
| else: | |
| ps_process = self.popen(['ps', '-eo', 'pid,comm'], | |
| stdout=self.PIPE, stderr=self.PIPE) | |
| stdout, _ = ps_process.communicate() | |
| for line in stdout.splitlines(): | |
| # In some cases the line can contain one or more | |
| # leading white-spaces, so strip it before split. | |
| pid, process_name = line.strip().split(' ', 1) | |
| processes.append([process_name, pid]) | |
| return processes | |
| def running_pids(self, process_name_filter=None): | |
| if not process_name_filter: | |
| process_name_filter = lambda process_name: True | |
| running_pids = [] | |
| for line in self._running_processes(): | |
| try: | |
| process_name = line[0] | |
| pid = line[1] | |
| if process_name_filter(process_name): | |
| running_pids.append(int(pid)) | |
| except (ValueError, IndexError): | |
| pass | |
| return sorted(running_pids) | |
| def process_dump(self): | |
| ps_process = None | |
| if sys.platform in ('win32', 'cygwin'): | |
| ps_process = self.popen( | |
| ['wmic', 'process', 'get', | |
| 'ProcessId,ParentProcessId,CommandLine'], | |
| stdout=self.PIPE, stderr=self.PIPE) | |
| else: | |
| ps_process = self.popen(['ps', 'aux'], stdout=self.PIPE, stderr=self.PIPE) | |
| stdout, _ = ps_process.communicate() | |
| return [line.strip() for line in stdout.splitlines()] | |
| def wait_newest(self, process_name_filter=None): | |
| if not process_name_filter: | |
| process_name_filter = lambda process_name: True | |
| running_pids = self.running_pids(process_name_filter) | |
| if not running_pids: | |
| return | |
| pid = running_pids[-1] | |
| while self.check_running_pid(pid): | |
| time.sleep(0.25) | |
| def wait_limited(self, pid, limit_in_seconds=None, check_frequency_in_seconds=None): | |
| seconds_left = limit_in_seconds or 10 | |
| sleep_length = check_frequency_in_seconds or 1 | |
| while seconds_left > 0 and self.check_running_pid(pid): | |
| seconds_left -= sleep_length | |
| time.sleep(sleep_length) | |
| def interrupt(self, pid): | |
| interrupt_signal = signal.SIGINT | |
| # Note: The python docs seem to suggest that on Windows, we may want to use | |
| # signal.CTRL_C_EVENT (http://docs.python.org/2/library/signal.html), but | |
| # it appears that signal.SIGINT also appears to work on Windows. | |
| try: | |
| os.kill(pid, interrupt_signal) | |
| except OSError: | |
| # Silently ignore when the pid doesn't exist. | |
| # It's impossible for callers to avoid race conditions with process shutdown. | |
| pass | |
| # Error handlers do not need to be static methods once all callers are | |
| # updated to use an Executive object. | |
| @staticmethod | |
| def default_error_handler(error): | |
| raise error | |
| @staticmethod | |
| def ignore_error(error): | |
| pass | |
| def _compute_stdin(self, input): | |
| """Returns (stdin, string_to_communicate)""" | |
| # FIXME: We should be returning /dev/null for stdin | |
| # or closing stdin after process creation to prevent | |
| # child processes from getting input from the user. | |
| if not input: | |
| return (None, None) | |
| if hasattr(input, 'read'): # Check if the input is a file. | |
| return (input, None) # Assume the file is in the right encoding. | |
| # Popen in Python 2.5 and before does not automatically encode unicode objects. | |
| # http://bugs.python.org/issue5290 | |
| # See https://bugs.webkit.org/show_bug.cgi?id=37528 | |
| # for an example of a regression caused by passing a unicode string directly. | |
| # FIXME: We may need to encode differently on different platforms. | |
| if isinstance(input, unicode): | |
| input = input.encode(self._child_process_encoding()) | |
| return (self.PIPE, input) | |
| def command_for_printing(self, args): | |
| """Returns a print-ready string representing command args. | |
| The string should be copy/paste ready for execution in a shell. | |
| """ | |
| args = self._stringify_args(args) | |
| escaped_args = [] | |
| for arg in args: | |
| if isinstance(arg, unicode): | |
| # Escape any non-ascii characters for easy copy/paste | |
| arg = arg.encode('unicode_escape') | |
| # FIXME: Do we need to fix quotes here? | |
| escaped_args.append(arg) | |
| return ' '.join(escaped_args) | |
| def run_command(self, | |
| args, | |
| cwd=None, | |
| env=None, | |
| input=None, | |
| timeout_seconds=None, | |
| error_handler=None, | |
| return_exit_code=False, | |
| return_stderr=True, | |
| decode_output=True, debug_logging=True): | |
| """Popen wrapper for convenience and to work around python bugs.""" | |
| assert isinstance(args, list) or isinstance(args, tuple) | |
| start_time = time.time() | |
| stdin, string_to_communicate = self._compute_stdin(input) | |
| stderr = self.STDOUT if return_stderr else None | |
| process = self.popen(args, | |
| stdin=stdin, | |
| stdout=self.PIPE, | |
| stderr=stderr, | |
| cwd=cwd, | |
| env=env, | |
| close_fds=self._should_close_fds()) | |
| if timeout_seconds: | |
| timer = threading.Timer(timeout_seconds, process.kill) | |
| timer.start() | |
| output = process.communicate(string_to_communicate)[0] | |
| # run_command automatically decodes to unicode() unless explicitly told not to. | |
| if decode_output: | |
| output = output.decode(self._child_process_encoding(), errors='replace') | |
| # wait() is not threadsafe and can throw OSError due to: | |
| # http://bugs.python.org/issue1731717 | |
| exit_code = process.wait() | |
| if timeout_seconds: | |
| timer.cancel() | |
| if debug_logging: | |
| _log.debug('"%s" took %.2fs', self.command_for_printing(args), time.time() - start_time) | |
| if return_exit_code: | |
| return exit_code | |
| if exit_code: | |
| script_error = ScriptError(script_args=args, | |
| exit_code=exit_code, | |
| output=output, | |
| cwd=cwd) | |
| (error_handler or self.default_error_handler)(script_error) | |
| return output | |
| def _child_process_encoding(self): | |
| # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW | |
| # to launch subprocesses, so we have to encode arguments using the | |
| # current code page. | |
| if sys.platform == 'win32' and sys.version < '3': | |
| return 'mbcs' | |
| # All other platforms use UTF-8. | |
| # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands | |
| # which will expect arguments to be encoded using the current code | |
| # page. | |
| return 'utf-8' | |
| def _should_encode_child_process_arguments(self): | |
| # Cygwin's Python's os.execv doesn't support unicode command | |
| # arguments, and neither does Cygwin's execv itself. | |
| if sys.platform == 'cygwin': | |
| return True | |
| # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW | |
| # to launch subprocesses, so we have to encode arguments using the | |
| # current code page. | |
| if sys.platform == 'win32' and sys.version < '3': | |
| return True | |
| return False | |
| def _encode_argument_if_needed(self, argument): | |
| if not self._should_encode_child_process_arguments(): | |
| return argument | |
| return argument.encode(self._child_process_encoding()) | |
| def _stringify_args(self, args): | |
| # Popen will throw an exception if args are non-strings (like int()) | |
| string_args = map(unicode, args) | |
| # The Windows implementation of Popen cannot handle unicode strings. :( | |
| return map(self._encode_argument_if_needed, string_args) | |
| def popen(self, args, **kwargs): | |
| assert not kwargs.get('shell') | |
| string_args = self._stringify_args(args) | |
| return subprocess.Popen(string_args, **kwargs) | |
| def call(self, args, **kwargs): | |
| return subprocess.call(self._stringify_args(args), **kwargs) | |
| def run_in_parallel(self, command_lines_and_cwds, processes=None): | |
| """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples.""" | |
| assert len(command_lines_and_cwds) | |
| return self.map(_run_command_thunk, command_lines_and_cwds, processes) | |
| def map(self, thunk, arglist, processes=None): | |
| if sys.platform in ('cygwin', 'win32') or len(arglist) == 1: | |
| return map(thunk, arglist) | |
| pool = multiprocessing.Pool(processes=(processes or multiprocessing.cpu_count())) | |
| try: | |
| return pool.map(thunk, arglist) | |
| finally: | |
| pool.close() | |
| pool.join() | |
| def _run_command_thunk(cmd_line_and_cwd): | |
| # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool. | |
| (cmd_line, cwd) = cmd_line_and_cwd | |
| proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| stdout, stderr = proc.communicate() | |
| return (proc.returncode, stdout, stderr) |