| #!/usr/bin/env python | |
| # Copyright 2016 the V8 project authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """ | |
| V8 correctness fuzzer launcher script. | |
| """ | |
| import argparse | |
| import hashlib | |
| import itertools | |
| import json | |
| import os | |
| import random | |
| import re | |
| import sys | |
| import traceback | |
| import v8_commands | |
| import v8_suppressions | |
| CONFIGS = dict( | |
| default=[ | |
| '--suppress-asm-messages', | |
| ], | |
| ignition=[ | |
| '--turbo-filter=~', | |
| '--noopt', | |
| '--suppress-asm-messages', | |
| ], | |
| ignition_asm=[ | |
| '--turbo-filter=~', | |
| '--noopt', | |
| '--validate-asm', | |
| '--stress-validate-asm', | |
| '--suppress-asm-messages', | |
| ], | |
| ignition_eager=[ | |
| '--turbo-filter=~', | |
| '--noopt', | |
| '--no-lazy', | |
| '--no-lazy-inner-functions', | |
| '--suppress-asm-messages', | |
| ], | |
| ignition_turbo=[ | |
| '--suppress-asm-messages', | |
| ], | |
| ignition_turbo_opt=[ | |
| '--always-opt', | |
| '--suppress-asm-messages', | |
| ], | |
| ignition_turbo_opt_eager=[ | |
| '--always-opt', | |
| '--no-lazy', | |
| '--no-lazy-inner-functions', | |
| '--suppress-asm-messages', | |
| ], | |
| slow_path=[ | |
| '--force-slow-path', | |
| '--suppress-asm-messages', | |
| ], | |
| slow_path_opt=[ | |
| '--always-opt', | |
| '--force-slow-path', | |
| '--suppress-asm-messages', | |
| ], | |
| trusted=[ | |
| '--no-untrusted-code-mitigations', | |
| '--suppress-asm-messages', | |
| ], | |
| trusted_opt=[ | |
| '--always-opt', | |
| '--no-untrusted-code-mitigations', | |
| '--suppress-asm-messages', | |
| ], | |
| ) | |
| # Additional flag experiments. List of tuples like | |
| # (<likelihood to use flags in [0,1)>, <flag>). | |
| ADDITIONAL_FLAGS = [ | |
| (0.1, '--stress-marking=100'), | |
| (0.1, '--stress-scavenge=100'), | |
| (0.1, '--stress-compaction-random'), | |
| (0.1, '--random-gc-interval=2000'), | |
| ] | |
| # Timeout in seconds for one d8 run. | |
| TIMEOUT = 3 | |
| # Return codes. | |
| RETURN_PASS = 0 | |
| RETURN_FAIL = 2 | |
| BASE_PATH = os.path.dirname(os.path.abspath(__file__)) | |
| PREAMBLE = [ | |
| os.path.join(BASE_PATH, 'v8_mock.js'), | |
| os.path.join(BASE_PATH, 'v8_suppressions.js'), | |
| ] | |
| ARCH_MOCKS = os.path.join(BASE_PATH, 'v8_mock_archs.js') | |
| FLAGS = ['--abort_on_stack_or_string_length_overflow', '--expose-gc', | |
| '--allow-natives-syntax', '--invoke-weak-callbacks', '--omit-quit', | |
| '--es-staging'] | |
| SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64'] | |
| # Output for suppressed failure case. | |
| FAILURE_HEADER_TEMPLATE = """# | |
| # V8 correctness failure | |
| # V8 correctness configs: %(configs)s | |
| # V8 correctness sources: %(source_key)s | |
| # V8 correctness suppression: %(suppression)s | |
| """ | |
| # Extended output for failure case. The 'CHECK' is for the minimizer. | |
| FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """# | |
| # CHECK | |
| # | |
| # Compared %(first_config_label)s with %(second_config_label)s | |
| # | |
| # Flags of %(first_config_label)s: | |
| %(first_config_flags)s | |
| # Flags of %(second_config_label)s: | |
| %(second_config_flags)s | |
| # | |
| # Difference: | |
| %(difference)s | |
| # | |
| # Source file: | |
| %(source)s | |
| # | |
| ### Start of configuration %(first_config_label)s: | |
| %(first_config_output)s | |
| ### End of configuration %(first_config_label)s | |
| # | |
| ### Start of configuration %(second_config_label)s: | |
| %(second_config_output)s | |
| ### End of configuration %(second_config_label)s | |
| """ | |
| FUZZ_TEST_RE = re.compile(r'.*fuzz(-\d+\.js)') | |
| SOURCE_RE = re.compile(r'print\("v8-foozzie source: (.*)"\);') | |
| # The number of hex digits used from the hash of the original source file path. | |
| # Keep the number small to avoid duplicate explosion. | |
| ORIGINAL_SOURCE_HASH_LENGTH = 3 | |
| # Placeholder string if no original source file could be determined. | |
| ORIGINAL_SOURCE_DEFAULT = 'none' | |
| def infer_arch(d8): | |
| """Infer the V8 architecture from the build configuration next to the | |
| executable. | |
| """ | |
| with open(os.path.join(os.path.dirname(d8), 'v8_build_config.json')) as f: | |
| arch = json.load(f)['v8_current_cpu'] | |
| return 'ia32' if arch == 'x86' else arch | |
| def parse_args(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument( | |
| '--random-seed', type=int, required=True, | |
| help='random seed passed to both runs') | |
| parser.add_argument( | |
| '--first-config', help='first configuration', default='ignition') | |
| parser.add_argument( | |
| '--second-config', help='second configuration', default='ignition_turbo') | |
| parser.add_argument( | |
| '--first-d8', default='d8', | |
| help='optional path to first d8 executable, ' | |
| 'default: bundled in the same directory as this script') | |
| parser.add_argument( | |
| '--second-d8', | |
| help='optional path to second d8 executable, default: same as first') | |
| parser.add_argument('testcase', help='path to test case') | |
| options = parser.parse_args() | |
| # Ensure we have a test case. | |
| assert (os.path.exists(options.testcase) and | |
| os.path.isfile(options.testcase)), ( | |
| 'Test case %s doesn\'t exist' % options.testcase) | |
| # Use first d8 as default for second d8. | |
| options.second_d8 = options.second_d8 or options.first_d8 | |
| # Ensure absolute paths. | |
| if not os.path.isabs(options.first_d8): | |
| options.first_d8 = os.path.join(BASE_PATH, options.first_d8) | |
| if not os.path.isabs(options.second_d8): | |
| options.second_d8 = os.path.join(BASE_PATH, options.second_d8) | |
| # Ensure executables exist. | |
| assert os.path.exists(options.first_d8) | |
| assert os.path.exists(options.second_d8) | |
| # Infer architecture from build artifacts. | |
| options.first_arch = infer_arch(options.first_d8) | |
| options.second_arch = infer_arch(options.second_d8) | |
| # Ensure we make a sane comparison. | |
| if (options.first_arch == options.second_arch and | |
| options.first_config == options.second_config): | |
| parser.error('Need either arch or config difference.') | |
| assert options.first_arch in SUPPORTED_ARCHS | |
| assert options.second_arch in SUPPORTED_ARCHS | |
| assert options.first_config in CONFIGS | |
| assert options.second_config in CONFIGS | |
| return options | |
| def get_meta_data(content): | |
| """Extracts original-source-file paths from test case content.""" | |
| sources = [] | |
| for line in content.splitlines(): | |
| match = SOURCE_RE.match(line) | |
| if match: | |
| sources.append(match.group(1)) | |
| return {'sources': sources} | |
| def content_bailout(content, ignore_fun): | |
| """Print failure state and return if ignore_fun matches content.""" | |
| bug = (ignore_fun(content) or '').strip() | |
| if bug: | |
| print FAILURE_HEADER_TEMPLATE % dict( | |
| configs='', source_key='', suppression=bug) | |
| return True | |
| return False | |
| def pass_bailout(output, step_number): | |
| """Print info and return if in timeout or crash pass states.""" | |
| if output.HasTimedOut(): | |
| # Dashed output, so that no other clusterfuzz tools can match the | |
| # words timeout or crash. | |
| print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number | |
| return True | |
| if output.HasCrashed(): | |
| print '# V8 correctness - C-R-A-S-H %d' % step_number | |
| return True | |
| return False | |
| def fail_bailout(output, ignore_by_output_fun): | |
| """Print failure state and return if ignore_by_output_fun matches output.""" | |
| bug = (ignore_by_output_fun(output.stdout) or '').strip() | |
| if bug: | |
| print FAILURE_HEADER_TEMPLATE % dict( | |
| configs='', source_key='', suppression=bug) | |
| return True | |
| return False | |
| def main(): | |
| options = parse_args() | |
| rng = random.Random(options.random_seed) | |
| # Suppressions are architecture and configuration specific. | |
| suppress = v8_suppressions.get_suppression( | |
| options.first_arch, options.first_config, | |
| options.second_arch, options.second_config, | |
| ) | |
| # Static bailout based on test case content or metadata. | |
| with open(options.testcase) as f: | |
| content = f.read() | |
| if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): | |
| return RETURN_FAIL | |
| if content_bailout(content, suppress.ignore_by_content): | |
| return RETURN_FAIL | |
| # Set up runtime arguments. | |
| common_flags = FLAGS + ['--random-seed', str(options.random_seed)] | |
| first_config_flags = common_flags + CONFIGS[options.first_config] | |
| second_config_flags = common_flags + CONFIGS[options.second_config] | |
| # Add additional flags to second config based on experiment percentages. | |
| for p, flag in ADDITIONAL_FLAGS: | |
| if rng.random() < p: | |
| second_config_flags.append(flag) | |
| def run_d8(d8, config_flags): | |
| preamble = PREAMBLE[:] | |
| if options.first_arch != options.second_arch: | |
| preamble.append(ARCH_MOCKS) | |
| args = [d8] + config_flags + preamble + [options.testcase] | |
| print " ".join(args) | |
| if d8.endswith('.py'): | |
| # Wrap with python in tests. | |
| args = [sys.executable] + args | |
| return v8_commands.Execute( | |
| args, | |
| cwd=os.path.dirname(os.path.abspath(options.testcase)), | |
| timeout=TIMEOUT, | |
| ) | |
| first_config_output = run_d8(options.first_d8, first_config_flags) | |
| # Early bailout based on first run's output. | |
| if pass_bailout(first_config_output, 1): | |
| return RETURN_PASS | |
| second_config_output = run_d8(options.second_d8, second_config_flags) | |
| # Bailout based on second run's output. | |
| if pass_bailout(second_config_output, 2): | |
| return RETURN_PASS | |
| difference, source = suppress.diff( | |
| first_config_output.stdout, second_config_output.stdout) | |
| if source: | |
| source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH] | |
| else: | |
| source = ORIGINAL_SOURCE_DEFAULT | |
| source_key = ORIGINAL_SOURCE_DEFAULT | |
| if difference: | |
| # Only bail out due to suppressed output if there was a difference. If a | |
| # suppression doesn't show up anymore in the statistics, we might want to | |
| # remove it. | |
| if fail_bailout(first_config_output, suppress.ignore_by_output1): | |
| return RETURN_FAIL | |
| if fail_bailout(second_config_output, suppress.ignore_by_output2): | |
| return RETURN_FAIL | |
| # The first three entries will be parsed by clusterfuzz. Format changes | |
| # will require changes on the clusterfuzz side. | |
| first_config_label = '%s,%s' % (options.first_arch, options.first_config) | |
| second_config_label = '%s,%s' % (options.second_arch, options.second_config) | |
| print (FAILURE_TEMPLATE % dict( | |
| configs='%s:%s' % (first_config_label, second_config_label), | |
| source_key=source_key, | |
| suppression='', # We can't tie bugs to differences. | |
| first_config_label=first_config_label, | |
| second_config_label=second_config_label, | |
| first_config_flags=' '.join(first_config_flags), | |
| second_config_flags=' '.join(second_config_flags), | |
| first_config_output= | |
| first_config_output.stdout.decode('utf-8', 'replace'), | |
| second_config_output= | |
| second_config_output.stdout.decode('utf-8', 'replace'), | |
| source=source, | |
| difference=difference.decode('utf-8', 'replace'), | |
| )).encode('utf-8', 'replace') | |
| return RETURN_FAIL | |
| # TODO(machenbach): Figure out if we could also return a bug in case there's | |
| # no difference, but one of the line suppressions has matched - and without | |
| # the match there would be a difference. | |
| print '# V8 correctness - pass' | |
| return RETURN_PASS | |
| if __name__ == "__main__": | |
| try: | |
| result = main() | |
| except SystemExit: | |
| # Make sure clusterfuzz reports internal errors and wrong usage. | |
| # Use one label for all internal and usage errors. | |
| print FAILURE_HEADER_TEMPLATE % dict( | |
| configs='', source_key='', suppression='wrong_usage') | |
| result = RETURN_FAIL | |
| except MemoryError: | |
| # Running out of memory happens occasionally but is not actionable. | |
| print '# V8 correctness - pass' | |
| result = RETURN_PASS | |
| except Exception as e: | |
| print FAILURE_HEADER_TEMPLATE % dict( | |
| configs='', source_key='', suppression='internal_error') | |
| print '# Internal error: %s' % e | |
| traceback.print_exc(file=sys.stdout) | |
| result = RETURN_FAIL | |
| sys.exit(result) |