| #!/usr/bin/python |
| |
| from __future__ import print_function |
| |
| import argparse |
| import getpass |
| import os |
| import os.path |
| import re |
| import select |
| import sys |
| import subprocess |
| |
| _COMMON_SYNC_OPTS = "-avzh --delete" |
| _COMMON_EXCLUDE_OPTS = "--exclude=DerivedData --exclude=.svn --exclude=.git --exclude=llvm-build/Release+Asserts" |
| |
| |
| def normalize_configuration(config_text): |
| if not config_text: |
| return "debug" |
| |
| config_lower = config_text.lower() |
| if config_lower in ["debug", "release"]: |
| return config_lower |
| else: |
| raise Exception("unknown configuration specified: %s" % config_text) |
| |
| |
| def parse_args(): |
| DEFAULT_REMOTE_ROOT_DIR = "/mnt/ssd/work/macosx.sync" |
| DEFAULT_REMOTE_HOSTNAME = "tfiala2.mtv.corp.google.com" |
| OPTIONS_FILENAME = ".remote-build.conf" |
| DEFAULT_SSH_PORT = "22" |
| |
| parser = argparse.ArgumentParser(fromfile_prefix_chars='@') |
| |
| parser.add_argument( |
| "--configuration", |
| "-c", |
| help="specify configuration (Debug, Release)", |
| default=normalize_configuration( |
| os.environ.get( |
| 'CONFIGURATION', |
| 'Debug'))) |
| parser.add_argument( |
| "--debug", "-d", |
| action="store_true", |
| help="help debug the remote-build script by adding extra logging") |
| parser.add_argument( |
| "--local-lldb-dir", "-l", metavar="DIR", |
| help="specify local lldb directory (Xcode layout assumed for llvm/clang)", |
| default=os.getcwd()) |
| parser.add_argument( |
| "--port", "-p", |
| help="specify the port ssh should use to connect to the remote side", |
| default=DEFAULT_SSH_PORT) |
| parser.add_argument( |
| "--remote-address", "-r", metavar="REMOTE-ADDR", |
| help="specify the dns name or ip address of the remote linux system", |
| default=DEFAULT_REMOTE_HOSTNAME) |
| parser.add_argument( |
| "--remote-dir", metavar="DIR", |
| help="specify the root of the linux source/build dir", |
| default=DEFAULT_REMOTE_ROOT_DIR) |
| parser.add_argument( |
| "--user", "-u", help="specify the user name for the remote system", |
| default=getpass.getuser()) |
| parser.add_argument( |
| "--xcode-action", |
| "-x", |
| help="$(ACTION) from Xcode", |
| nargs='?', |
| default=None) |
| |
| command_line_args = sys.argv[1:] |
| if os.path.exists(OPTIONS_FILENAME): |
| # Prepend the file so that command line args override the file |
| # contents. |
| command_line_args.insert(0, "@%s" % OPTIONS_FILENAME) |
| |
| return parser.parse_args(command_line_args) |
| |
| |
| def maybe_create_remote_root_dir(args): |
| commandline = [ |
| "ssh", |
| "-p", args.port, |
| "%s@%s" % (args.user, args.remote_address), |
| "mkdir", |
| "-p", |
| args.remote_dir] |
| print("create remote root dir command:\n{}".format(commandline)) |
| return subprocess.call(commandline) |
| |
| |
| def init_with_args(args): |
| # Expand any user directory specs in local-side source dir (on MacOSX). |
| args.local_lldb_dir = os.path.expanduser(args.local_lldb_dir) |
| |
| # Append the configuration type to the remote build dir. |
| args.configuration = normalize_configuration(args.configuration) |
| args.remote_build_dir = os.path.join( |
| args.remote_dir, |
| "build-%s" % args.configuration) |
| |
| # We assume the local lldb directory is really named 'lldb'. |
| # This is because on the remote end, the local lldb root dir |
| # is copied over underneath llvm/tools and will be named there |
| # whatever it is named locally. The remote build will assume |
| # is is called lldb. |
| if os.path.basename(args.local_lldb_dir) != 'lldb': |
| raise Exception( |
| "local lldb root needs to be called 'lldb' but was {} instead" |
| .format(os.path.basename(args.local_lldb_dir))) |
| |
| args.lldb_dir_relative_regex = re.compile( |
| "%s/llvm/tools/lldb/" % args.remote_dir) |
| args.llvm_dir_relative_regex = re.compile("%s/" % args.remote_dir) |
| |
| print("Xcode action:", args.xcode_action) |
| |
| # Ensure the remote directory exists. |
| result = maybe_create_remote_root_dir(args) |
| if result == 0: |
| print("using remote root dir: %s" % args.remote_dir) |
| else: |
| print("remote root dir doesn't exist and could not be created, " |
| + "error code:", result) |
| return False |
| |
| return True |
| |
| |
| def sync_llvm(args): |
| commandline = ["rsync"] |
| commandline.extend(_COMMON_SYNC_OPTS.split()) |
| commandline.extend(_COMMON_EXCLUDE_OPTS.split()) |
| commandline.append("--exclude=/llvm/tools/lldb") |
| commandline.extend(["-e", "ssh -p {}".format(args.port)]) |
| commandline.extend([ |
| "%s/llvm" % args.local_lldb_dir, |
| "%s@%s:%s" % (args.user, args.remote_address, args.remote_dir)]) |
| if args.debug: |
| print("going to execute llvm sync: {}".format(commandline)) |
| return subprocess.call(commandline) |
| |
| |
| def sync_lldb(args): |
| commandline = ["rsync"] |
| commandline.extend(_COMMON_SYNC_OPTS.split()) |
| commandline.extend(_COMMON_EXCLUDE_OPTS.split()) |
| commandline.append("--exclude=/lldb/llvm") |
| commandline.extend(["-e", "ssh -p {}".format(args.port)]) |
| commandline.extend([args.local_lldb_dir, "%s@%s:%s/llvm/tools" % |
| (args.user, args.remote_address, args.remote_dir)]) |
| if args.debug: |
| print("going to execute lldb sync: {}".format(commandline)) |
| return subprocess.call(commandline) |
| |
| |
| def build_cmake_command(args): |
| # args.remote_build_dir |
| # args.configuration in ('release', 'debug') |
| |
| if args.configuration == 'debug-optimized': |
| build_type_name = "RelWithDebInfo" |
| elif args.configuration == 'release': |
| build_type_name = "Release" |
| else: |
| build_type_name = "Debug" |
| |
| ld_flags = "\"-lstdc++ -lm\"" |
| |
| install_dir = os.path.join( |
| args.remote_build_dir, "..", "install-{}".format(args.configuration)) |
| |
| command_line = [ |
| "cmake", |
| "-GNinja", |
| "-DCMAKE_CXX_COMPILER=clang", |
| "-DCMAKE_C_COMPILER=clang", |
| # "-DCMAKE_CXX_FLAGS=%s" % cxx_flags, |
| "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ld_flags, |
| "-DCMAKE_EXE_LINKER_FLAGS=%s" % ld_flags, |
| "-DCMAKE_INSTALL_PREFIX:PATH=%s" % install_dir, |
| "-DCMAKE_BUILD_TYPE=%s" % build_type_name, |
| "-Wno-dev", |
| os.path.join("..", "llvm") |
| ] |
| |
| return command_line |
| |
| |
| def maybe_configure(args): |
| commandline = [ |
| "ssh", |
| "-p", args.port, |
| "%s@%s" % (args.user, args.remote_address), |
| "cd", args.remote_dir, "&&", |
| "mkdir", "-p", args.remote_build_dir, "&&", |
| "cd", args.remote_build_dir, "&&" |
| ] |
| commandline.extend(build_cmake_command(args)) |
| |
| if args.debug: |
| print("configure command: {}".format(commandline)) |
| |
| return subprocess.call(commandline) |
| |
| |
| def filter_build_line(args, line): |
| lldb_relative_line = args.lldb_dir_relative_regex.sub('', line) |
| if len(lldb_relative_line) != len(line): |
| # We substituted - return the modified line |
| return lldb_relative_line |
| |
| # No match on lldb path (longer on linux than llvm path). Try |
| # the llvm path match. |
| return args.llvm_dir_relative_regex.sub('', line) |
| |
| |
| def run_remote_build_command(args, build_command_list): |
| commandline = [ |
| "ssh", |
| "-p", args.port, |
| "%s@%s" % (args.user, args.remote_address), |
| "cd", args.remote_build_dir, "&&"] |
| commandline.extend(build_command_list) |
| |
| if args.debug: |
| print("running remote build command: {}".format(commandline)) |
| |
| proc = subprocess.Popen( |
| commandline, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE) |
| |
| # Filter stdout/stderr output for file path mapping. |
| # We do this to enable Xcode to see filenames relative to the |
| # MacOSX-side directory structure. |
| while True: |
| reads = [proc.stdout.fileno(), proc.stderr.fileno()] |
| select_result = select.select(reads, [], []) |
| |
| for fd in select_result[0]: |
| if fd == proc.stdout.fileno(): |
| line = proc.stdout.readline() |
| display_line = filter_build_line(args, line.rstrip()) |
| if display_line and len(display_line) > 0: |
| print(display_line) |
| elif fd == proc.stderr.fileno(): |
| line = proc.stderr.readline() |
| display_line = filter_build_line(args, line.rstrip()) |
| if display_line and len(display_line) > 0: |
| print(display_line, file=sys.stderr) |
| |
| proc_retval = proc.poll() |
| if proc_retval is not None: |
| # Process stopped. Drain output before finishing up. |
| |
| # Drain stdout. |
| while True: |
| line = proc.stdout.readline() |
| if line: |
| display_line = filter_build_line(args, line.rstrip()) |
| if display_line and len(display_line) > 0: |
| print(display_line) |
| else: |
| break |
| |
| # Drain stderr. |
| while True: |
| line = proc.stderr.readline() |
| if line: |
| display_line = filter_build_line(args, line.rstrip()) |
| if display_line and len(display_line) > 0: |
| print(display_line, file=sys.stderr) |
| else: |
| break |
| |
| return proc_retval |
| |
| |
| def build(args): |
| return run_remote_build_command(args, ["time", "ninja"]) |
| |
| |
| def clean(args): |
| return run_remote_build_command(args, ["ninja", "clean"]) |
| |
| |
| if __name__ == "__main__": |
| # Handle arg parsing. |
| args = parse_args() |
| |
| # Initialize the system. |
| if not init_with_args(args): |
| exit(1) |
| |
| # Sync over llvm and clang source. |
| sync_llvm(args) |
| |
| # Sync over lldb source. |
| sync_lldb(args) |
| |
| # Configure the remote build if it's not already. |
| maybe_configure(args) |
| |
| if args.xcode_action == 'clean': |
| exit(clean(args)) |
| else: |
| exit(build(args)) |
