| #!/usr/bin/env python |
| # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Test harness for chromium clang tools.""" |
| |
| import argparse |
| import difflib |
| import glob |
| import json |
| import os |
| import os.path |
| import shutil |
| import subprocess |
| import sys |
| |
| |
| def _RunGit(args): |
| if sys.platform == 'win32': |
| args = ['git.bat'] + args |
| else: |
| args = ['git'] + args |
| subprocess.check_call(args) |
| |
| |
| def _GenerateCompileCommands(files, include_paths): |
| """Returns a JSON string containing a compilation database for the input.""" |
| # Note: in theory, backslashes in the compile DB should work but the tools |
| # that write compile DBs and the tools that read them don't agree on the |
| # escaping convention: https://llvm.org/bugs/show_bug.cgi?id=19687 |
| files = [f.replace('\\', '/') for f in files] |
| include_path_flags = ' '.join('-I %s' % include_path.replace('\\', '/') |
| for include_path in include_paths) |
| return json.dumps([{'directory': os.path.dirname(f), |
| 'command': 'clang++ -std=c++14 -fsyntax-only %s -c %s' % ( |
| include_path_flags, os.path.basename(f)), |
| 'file': os.path.basename(f)} for f in files], indent=2) |
| |
| |
| def _NumberOfTestsToString(tests): |
| """Returns an English describing the number of tests.""" |
| return '%d test%s' % (tests, 's' if tests != 1 else '') |
| |
| |
| def _ApplyTool(tools_clang_scripts_directory, |
| tool_to_test, |
| tool_path, |
| tool_args, |
| test_directory_for_tool, |
| actual_files, |
| apply_edits): |
| try: |
| # Stage the test files in the git index. If they aren't staged, then |
| # run_tool.py will skip them when applying replacements. |
| args = ['add'] |
| args.extend(actual_files) |
| _RunGit(args) |
| |
| # Launch the following pipeline if |apply_edits| is True: |
| # run_tool.py ... | extract_edits.py | apply_edits.py ... |
| # Otherwise just the first step is done and the result is written to |
| # actual_files[0]. |
| processes = [] |
| args = ['python', |
| os.path.join(tools_clang_scripts_directory, 'run_tool.py')] |
| extra_run_tool_args_path = os.path.join(test_directory_for_tool, |
| 'run_tool.args') |
| if os.path.exists(extra_run_tool_args_path): |
| with open(extra_run_tool_args_path, 'r') as extra_run_tool_args_file: |
| extra_run_tool_args = extra_run_tool_args_file.readlines() |
| args.extend([arg.strip() for arg in extra_run_tool_args]) |
| args.extend(['--tool', tool_to_test, '-p', test_directory_for_tool]) |
| |
| if tool_path: |
| args.extend(['--tool-path', tool_path]) |
| if tool_args: |
| for arg in tool_args: |
| args.append('--tool-arg=%s' % arg) |
| |
| args.extend(actual_files) |
| processes.append(subprocess.Popen(args, stdout=subprocess.PIPE)) |
| |
| if apply_edits: |
| args = [ |
| 'python', |
| os.path.join(tools_clang_scripts_directory, 'extract_edits.py') |
| ] |
| processes.append(subprocess.Popen( |
| args, stdin=processes[-1].stdout, stdout=subprocess.PIPE)) |
| |
| args = [ |
| 'python', |
| os.path.join(tools_clang_scripts_directory, 'apply_edits.py'), '-p', |
| test_directory_for_tool |
| ] |
| processes.append(subprocess.Popen( |
| args, stdin=processes[-1].stdout, stdout=subprocess.PIPE)) |
| |
| # Wait for the pipeline to finish running + check exit codes. |
| stdout, _ = processes[-1].communicate() |
| for process in processes: |
| process.wait() |
| if process.returncode != 0: |
| print 'Failure while running the tool.' |
| return process.returncode |
| |
| if apply_edits: |
| # Reformat the resulting edits via: git cl format. |
| args = ['cl', 'format'] |
| args.extend(actual_files) |
| _RunGit(args) |
| else: |
| with open(actual_files[0], 'w') as output_file: |
| output_file.write(stdout) |
| |
| return 0 |
| |
| finally: |
| # No matter what, unstage the git changes we made earlier to avoid polluting |
| # the index. |
| args = ['reset', '--quiet', 'HEAD'] |
| args.extend(actual_files) |
| _RunGit(args) |
| |
| |
| def main(argv): |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| '--apply-edits', |
| action='store_true', |
| help='Applies the edits to the original test files and compares the ' |
| 'reformatted new files with the expected files.') |
| parser.add_argument( |
| '--tool-arg', nargs='?', action='append', |
| help='optional arguments passed to the tool') |
| parser.add_argument( |
| '--tool-path', nargs='?', |
| help='optional path to the tool directory') |
| parser.add_argument('tool_name', |
| nargs=1, |
| help='Clang tool to be tested.') |
| args = parser.parse_args(argv) |
| tool_to_test = args.tool_name[0] |
| print '\nTesting %s\n' % tool_to_test |
| tools_clang_scripts_directory = os.path.dirname(os.path.realpath(__file__)) |
| tools_clang_directory = os.path.dirname(tools_clang_scripts_directory) |
| test_directory_for_tool = os.path.join( |
| tools_clang_directory, tool_to_test, 'tests') |
| compile_database = os.path.join(test_directory_for_tool, |
| 'compile_commands.json') |
| source_files = glob.glob(os.path.join(test_directory_for_tool, |
| '*-original.cc')) |
| ext = 'cc' if args.apply_edits else 'txt' |
| actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc']) |
| for source_file in source_files] |
| expected_files = ['-'.join([source_file.rsplit('-', 1)[0], 'expected.' + ext]) |
| for source_file in source_files] |
| if not args.apply_edits and len(actual_files) != 1: |
| print 'Only one test file is expected for testing without apply-edits.' |
| return 1 |
| |
| include_paths = [] |
| include_paths.append( |
| os.path.realpath(os.path.join(tools_clang_directory, '../..'))) |
| # Many gtest and gmock headers expect to have testing/gtest/include and/or |
| # testing/gmock/include in the include search path. |
| include_paths.append( |
| os.path.realpath(os.path.join(tools_clang_directory, |
| '../..', |
| 'testing/gtest/include'))) |
| include_paths.append( |
| os.path.realpath(os.path.join(tools_clang_directory, |
| '../..', |
| 'testing/gmock/include'))) |
| |
| if len(actual_files) == 0: |
| print 'Tool "%s" does not have compatible test files.' % tool_to_test |
| return 1 |
| |
| # Set up the test environment. |
| for source, actual in zip(source_files, actual_files): |
| shutil.copyfile(source, actual) |
| # Generate a temporary compilation database to run the tool over. |
| with open(compile_database, 'w') as f: |
| f.write(_GenerateCompileCommands(actual_files, include_paths)) |
| |
| # Run the tool. |
| os.chdir(test_directory_for_tool) |
| exitcode = _ApplyTool(tools_clang_scripts_directory, tool_to_test, |
| args.tool_path, args.tool_arg, |
| test_directory_for_tool, actual_files, |
| args.apply_edits) |
| if (exitcode != 0): |
| return exitcode |
| |
| # Compare actual-vs-expected results. |
| passed = 0 |
| failed = 0 |
| for expected, actual in zip(expected_files, actual_files): |
| print '[ RUN ] %s' % os.path.relpath(actual) |
| expected_output = actual_output = None |
| with open(expected, 'r') as f: |
| expected_output = f.read().splitlines() |
| with open(actual, 'r') as f: |
| actual_output = f.read().splitlines() |
| if actual_output != expected_output: |
| failed += 1 |
| for line in difflib.unified_diff(expected_output, actual_output, |
| fromfile=os.path.relpath(expected), |
| tofile=os.path.relpath(actual)): |
| sys.stdout.write(line) |
| print '[ FAILED ] %s' % os.path.relpath(actual) |
| # Don't clean up the file on failure, so the results can be referenced |
| # more easily. |
| continue |
| print '[ OK ] %s' % os.path.relpath(actual) |
| passed += 1 |
| os.remove(actual) |
| |
| if failed == 0: |
| os.remove(compile_database) |
| |
| print '[==========] %s ran.' % _NumberOfTestsToString(len(source_files)) |
| if passed > 0: |
| print '[ PASSED ] %s.' % _NumberOfTestsToString(passed) |
| if failed > 0: |
| print '[ FAILED ] %s.' % _NumberOfTestsToString(failed) |
| return 1 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |
