blob: 18be227d98902972695fdd61fcb3bed60b6d8b32 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from os.path import join
import itertools
import json
import math
import multiprocessing
import os
import random
import shlex
import sys
import time
# Adds testrunner to the path hence it has to be imported at the beggining.
import base_runner
from testrunner.local import execution
from testrunner.local import progress
from testrunner.local import testsuite
from testrunner.local import utils
from testrunner.local import verbose
from testrunner.objects import context
DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]
# Double the timeout for these:
SLOW_ARCHS = ["arm",
class GCFuzzer(base_runner.BaseTestRunner):
def __init__(self, *args, **kwargs):
super(GCFuzzer, self).__init__(*args, **kwargs)
self.fuzzer_rng = None
def _add_parser_options(self, parser):
help="Prepended to each shell command used to run a test",
parser.add_option("--coverage", help=("Exponential test coverage "
"(range 0.0, 1.0) - 0.0: one test, 1.0 all tests (slow)"),
default=0.4, type="float")
parser.add_option("--coverage-lift", help=("Lifts test coverage for tests "
"with a low memory size reached (range 0, inf)"),
default=20, type="int")
parser.add_option("--dump-results-file", help="Dump maximum limit reached")
help="Additional flags to pass to each test command",
parser.add_option("--isolates", help="Whether to test isolates",
default=False, action="store_true")
parser.add_option("-j", help="The number of parallel tasks to run",
default=0, type="int")
parser.add_option("-p", "--progress",
help=("The style of progress indicator"
" (verbose, dots, color, mono)"),
parser.add_option("-t", "--timeout", help="Timeout in seconds",
default= -1, type="int")
parser.add_option("--random-seed", default=0,
help="Default seed for initializing random generator")
parser.add_option("--fuzzer-random-seed", default=0,
help="Default seed for initializing fuzzer random "
parser.add_option("--stress-compaction", default=False, action="store_true",
help="Enable stress_compaction_percentage flag")
parser.add_option("--distribution-factor1", help="DEPRECATED")
parser.add_option("--distribution-factor2", help="DEPRECATED")
parser.add_option("--distribution-mode", help="DEPRECATED")
parser.add_option("--seed", help="DEPRECATED")
return parser
def _process_options(self, options):
# Special processing of other options, sorted alphabetically.
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = shlex.split(options.extra_flags)
if options.j == 0:
options.j = multiprocessing.cpu_count()
while options.random_seed == 0:
options.random_seed = random.SystemRandom().randint(-2147483648,
while options.fuzzer_random_seed == 0:
options.fuzzer_random_seed = random.SystemRandom().randint(-2147483648,
self.fuzzer_rng = random.Random(options.fuzzer_random_seed)
return True
def _calculate_n_tests(self, m, options):
"""Calculates the number of tests from m points with exponential coverage.
The coverage is expected to be between 0.0 and 1.0.
The 'coverage lift' lifts the coverage for tests with smaller m values.
c = float(options.coverage)
l = float(options.coverage_lift)
return int(math.pow(m, (m * c + l) / (m + l)))
def _get_default_suite_names(self):
def _do_execute(self, suites, args, options):
print(">>> Running tests for %s.%s" % (self.build_config.arch,
# Populate context object.
timeout = options.timeout
if timeout == -1:
# Simulators are slow, therefore allow a longer default timeout.
if self.build_config.arch in SLOW_ARCHS:
timeout = 2 * TIMEOUT_DEFAULT;
timeout *= self.mode_options.timeout_scalefactor
ctx = context.Context(self.build_config.arch,
self.mode_options.flags, options.verbose,
timeout, options.isolates,
False, # Keep i18n on by default.
True, # No sorting of test cases.
0, # Don't rerun failing tests.
0, # No use of a rerun-failing-tests maximum.
False, # No no_harness mode.
False, # Don't use perf data.
False) # Coverage not supported.
num_tests = self._load_tests(args, options, suites, ctx)
if num_tests == 0:
print "No tests to run."
return 0
test_backup = dict(map(lambda s: (s, s.tests), suites))
print('>>> Collection phase')
for s in suites:
analysis_flags = ['--fuzzer-gc-analysis']
s.tests = map(lambda t: t.create_variant(t.variant, analysis_flags,
for t in s.tests:
t.cmd = t.get_command(ctx)
progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
runner = execution.Runner(suites, progress_indicator, ctx)
exit_code = runner.Run(options.j)
print('>>> Analysis phase')
test_results = dict()
for s in suites:
for t in s.tests:
# Skip failed tests.
if t.output_proc.has_unexpected_output(runner.outputs[t]):
print '%s failed, skipping' % t.path
max_limit = self._get_max_limit_reached(runner.outputs[t])
if max_limit:
test_results[t.path] = max_limit
runner = None
if options.dump_results_file:
with file("%s.%d.txt" % (options.dump_results_file, time.time()),
"w") as f:
num_tests = 0
for s in suites:
s.tests = []
for t in test_backup[s]:
max_percent = test_results.get(t.path, 0)
if not max_percent or max_percent < 1.0:
max_percent = int(max_percent)
subtests_count = self._calculate_n_tests(max_percent, options)
if options.verbose:
print ('%s [x%d] (max marking limit=%.02f)' %
(t.path, subtests_count, max_percent))
for i in xrange(0, subtests_count):
fuzzer_seed = self._next_fuzzer_seed()
fuzzing_flags = [
'--stress_marking', str(max_percent),
'--fuzzer_random_seed', str(fuzzer_seed),
if options.stress_compaction:
s.tests.append(t.create_variant(t.variant, fuzzing_flags, i))
for t in s.tests:
t.cmd = t.get_command(ctx)
num_tests += len(s.tests)
if num_tests == 0:
print "No tests to run."
return exit_code
print(">>> Fuzzing phase (%d test cases)" % num_tests)
progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
runner = execution.Runner(suites, progress_indicator, ctx)
return runner.Run(options.j) or exit_code
def _load_tests(self, args, options, suites, ctx):
# Find available test suites and read test cases from them.
variables = {
"arch": self.build_config.arch,
"asan": self.build_config.asan,
"byteorder": sys.byteorder,
"dcheck_always_on": self.build_config.dcheck_always_on,
"deopt_fuzzer": False,
"gc_fuzzer": True,
"gc_stress": False,
"gcov_coverage": self.build_config.gcov_coverage,
"isolates": options.isolates,
"mode": self.mode_options.status_mode,
"msan": self.build_config.msan,
"no_harness": False,
"no_i18n": self.build_config.no_i18n,
"no_snap": self.build_config.no_snap,
"novfp3": False,
"predictable": self.build_config.predictable,
"simulator": utils.UseSimulator(self.build_config.arch),
"simulator_run": False,
"system": utils.GuessOS(),
"tsan": self.build_config.tsan,
"ubsan_vptr": self.build_config.ubsan_vptr,
num_tests = 0
test_id = 0
for s in suites:
if len(args) > 0:
num_tests += len(s.tests)
for t in s.tests: = test_id
test_id += 1
return num_tests
# Parses test stdout and returns what was the highest reached percent of the
# incremental marking limit (0-100).
def _get_max_limit_reached(output):
if not output.stdout:
return None
for l in reversed(output.stdout.splitlines()):
if l.startswith('### Maximum marking limit reached ='):
return float(l.split()[6])
return None
def _next_fuzzer_seed(self):
fuzzer_seed = None
while not fuzzer_seed:
fuzzer_seed = self.fuzzer_rng.randint(-2147483648, 2147483647)
return fuzzer_seed
if __name__ == '__main__':