| #!/usr/bin/env python | |
| # | |
| # Copyright 2017 the V8 project authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| from collections import OrderedDict | |
| from os.path import join | |
| import multiprocessing | |
| import os | |
| import random | |
| import shlex | |
| import subprocess | |
| import sys | |
| import time | |
| # Adds testrunner to the path hence it has to be imported at the beggining. | |
| import base_runner | |
| from testrunner.local import execution | |
| from testrunner.local import progress | |
| from testrunner.local import testsuite | |
| from testrunner.local import utils | |
| from testrunner.local import verbose | |
| from testrunner.local.variants import ALL_VARIANTS | |
| from testrunner.objects import context | |
| from testrunner.objects import predictable | |
| from testrunner.testproc.execution import ExecutionProc | |
| from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc | |
| from testrunner.testproc.loader import LoadProc | |
| from testrunner.testproc.progress import (VerboseProgressIndicator, | |
| ResultsTracker, | |
| TestsCounter) | |
| from testrunner.testproc.rerun import RerunProc | |
| from testrunner.testproc.variant import VariantProc | |
| TIMEOUT_DEFAULT = 60 | |
| # Variants ordered by expected runtime (slowest first). | |
| VARIANTS = ["default"] | |
| MORE_VARIANTS = [ | |
| "stress", | |
| "stress_incremental_marking", | |
| "nooptimization", | |
| "stress_background_compile", | |
| "wasm_traps", | |
| ] | |
| VARIANT_ALIASES = { | |
| # The default for developer workstations. | |
| "dev": VARIANTS, | |
| # Additional variants, run on all bots. | |
| "more": MORE_VARIANTS, | |
| # Shortcut for the two above ("more" first - it has the longer running tests). | |
| "exhaustive": MORE_VARIANTS + VARIANTS, | |
| # Additional variants, run on a subset of bots. | |
| "extra": ["future", "liftoff", "trusted"], | |
| } | |
| GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", | |
| "--concurrent-recompilation-queue-length=64", | |
| "--concurrent-recompilation-delay=500", | |
| "--concurrent-recompilation"] | |
| # Double the timeout for these: | |
| SLOW_ARCHS = ["arm", | |
| "mips", | |
| "mipsel", | |
| "mips64", | |
| "mips64el", | |
| "s390", | |
| "s390x", | |
| "arm64"] | |
| PREDICTABLE_WRAPPER = os.path.join( | |
| base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py') | |
| class StandardTestRunner(base_runner.BaseTestRunner): | |
| def __init__(self, *args, **kwargs): | |
| super(StandardTestRunner, self).__init__(*args, **kwargs) | |
| self.sancov_dir = None | |
| def _get_default_suite_names(self): | |
| return ['default'] | |
| def _do_execute(self, suites, args, options): | |
| if options.swarming: | |
| # Swarming doesn't print how isolated commands are called. Lets make | |
| # this less cryptic by printing it ourselves. | |
| print ' '.join(sys.argv) | |
| if utils.GuessOS() == "macos": | |
| # TODO(machenbach): Temporary output for investigating hanging test | |
| # driver on mac. | |
| print "V8 related processes running on this host:" | |
| try: | |
| print subprocess.check_output( | |
| "ps -e | egrep 'd8|cctest|unittests'", shell=True) | |
| except Exception: | |
| pass | |
| return self._execute(args, options, suites) | |
| def _add_parser_options(self, parser): | |
| parser.add_option("--sancov-dir", | |
| help="Directory where to collect coverage data") | |
| parser.add_option("--cfi-vptr", | |
| help="Run tests with UBSAN cfi_vptr option.", | |
| default=False, action="store_true") | |
| parser.add_option("--novfp3", | |
| help="Indicates that V8 was compiled without VFP3" | |
| " support", | |
| default=False, action="store_true") | |
| parser.add_option("--cat", help="Print the source of the tests", | |
| default=False, action="store_true") | |
| parser.add_option("--slow-tests", | |
| help="Regard slow tests (run|skip|dontcare)", | |
| default="dontcare") | |
| parser.add_option("--pass-fail-tests", | |
| help="Regard pass|fail tests (run|skip|dontcare)", | |
| default="dontcare") | |
| parser.add_option("--gc-stress", | |
| help="Switch on GC stress mode", | |
| default=False, action="store_true") | |
| parser.add_option("--command-prefix", | |
| help="Prepended to each shell command used to run a" | |
| " test", | |
| default="") | |
| parser.add_option("--extra-flags", | |
| help="Additional flags to pass to each test command", | |
| action="append", default=[]) | |
| parser.add_option("--infra-staging", help="Use new test runner features", | |
| default=False, action="store_true") | |
| parser.add_option("--isolates", help="Whether to test isolates", | |
| default=False, action="store_true") | |
| parser.add_option("-j", help="The number of parallel tasks to run", | |
| default=0, type="int") | |
| parser.add_option("--no-harness", "--noharness", | |
| help="Run without test harness of a given suite", | |
| default=False, action="store_true") | |
| parser.add_option("--no-presubmit", "--nopresubmit", | |
| help='Skip presubmit checks (deprecated)', | |
| default=False, dest="no_presubmit", action="store_true") | |
| parser.add_option("--no-sorting", "--nosorting", | |
| help="Don't sort tests according to duration of last" | |
| " run.", | |
| default=False, dest="no_sorting", action="store_true") | |
| parser.add_option("--no-variants", "--novariants", | |
| help="Deprecated. " | |
| "Equivalent to passing --variants=default", | |
| default=False, dest="no_variants", action="store_true") | |
| parser.add_option("--variants", | |
| help="Comma-separated list of testing variants;" | |
| " default: \"%s\"" % ",".join(VARIANTS)) | |
| parser.add_option("--exhaustive-variants", | |
| default=False, action="store_true", | |
| help="Deprecated. " | |
| "Equivalent to passing --variants=exhaustive") | |
| parser.add_option("-p", "--progress", | |
| help=("The style of progress indicator" | |
| " (verbose, dots, color, mono)"), | |
| choices=progress.PROGRESS_INDICATORS.keys(), | |
| default="mono") | |
| parser.add_option("--quickcheck", default=False, action="store_true", | |
| help=("Quick check mode (skip slow tests)")) | |
| parser.add_option("--report", help="Print a summary of the tests to be" | |
| " run", | |
| default=False, action="store_true") | |
| parser.add_option("--json-test-results", | |
| help="Path to a file for storing json results.") | |
| parser.add_option("--flakiness-results", | |
| help="Path to a file for storing flakiness json.") | |
| parser.add_option("--rerun-failures-count", | |
| help=("Number of times to rerun each failing test case." | |
| " Very slow tests will be rerun only once."), | |
| default=0, type="int") | |
| parser.add_option("--rerun-failures-max", | |
| help="Maximum number of failing test cases to rerun.", | |
| default=100, type="int") | |
| parser.add_option("--dont-skip-slow-simulator-tests", | |
| help="Don't skip more slow tests when using a" | |
| " simulator.", | |
| default=False, action="store_true", | |
| dest="dont_skip_simulator_slow_tests") | |
| parser.add_option("--swarming", | |
| help="Indicates running test driver on swarming.", | |
| default=False, action="store_true") | |
| parser.add_option("--time", help="Print timing information after running", | |
| default=False, action="store_true") | |
| parser.add_option("-t", "--timeout", help="Timeout in seconds", | |
| default=TIMEOUT_DEFAULT, type="int") | |
| parser.add_option("--warn-unused", help="Report unused rules", | |
| default=False, action="store_true") | |
| parser.add_option("--junitout", help="File name of the JUnit output") | |
| parser.add_option("--junittestsuite", | |
| help="The testsuite name in the JUnit output file", | |
| default="v8tests") | |
| parser.add_option("--random-seed", default=0, dest="random_seed", | |
| help="Default seed for initializing random generator", | |
| type=int) | |
| parser.add_option("--random-seed-stress-count", default=1, type="int", | |
| dest="random_seed_stress_count", | |
| help="Number of runs with different random seeds") | |
| def _process_options(self, options): | |
| global VARIANTS | |
| if options.sancov_dir: | |
| self.sancov_dir = options.sancov_dir | |
| if not os.path.exists(self.sancov_dir): | |
| print("sancov-dir %s doesn't exist" % self.sancov_dir) | |
| raise base_runner.TestRunnerError() | |
| options.command_prefix = shlex.split(options.command_prefix) | |
| options.extra_flags = sum(map(shlex.split, options.extra_flags), []) | |
| if options.gc_stress: | |
| options.extra_flags += GC_STRESS_FLAGS | |
| if self.build_config.asan: | |
| options.extra_flags.append("--invoke-weak-callbacks") | |
| options.extra_flags.append("--omit-quit") | |
| if options.novfp3: | |
| options.extra_flags.append("--noenable-vfp3") | |
| if options.no_variants: # pragma: no cover | |
| print ("Option --no-variants is deprecated. " | |
| "Pass --variants=default instead.") | |
| assert not options.variants | |
| options.variants = "default" | |
| if options.exhaustive_variants: # pragma: no cover | |
| # TODO(machenbach): Switch infra to --variants=exhaustive after M65. | |
| print ("Option --exhaustive-variants is deprecated. " | |
| "Pass --variants=exhaustive instead.") | |
| # This is used on many bots. It includes a larger set of default | |
| # variants. | |
| # Other options for manipulating variants still apply afterwards. | |
| assert not options.variants | |
| options.variants = "exhaustive" | |
| if options.quickcheck: | |
| assert not options.variants | |
| options.variants = "stress,default" | |
| options.slow_tests = "skip" | |
| options.pass_fail_tests = "skip" | |
| if self.build_config.predictable: | |
| options.variants = "default" | |
| options.extra_flags.append("--predictable") | |
| options.extra_flags.append("--verify_predictable") | |
| options.extra_flags.append("--no-inline-new") | |
| # Add predictable wrapper to command prefix. | |
| options.command_prefix = ( | |
| [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) | |
| # TODO(machenbach): Figure out how to test a bigger subset of variants on | |
| # msan. | |
| if self.build_config.msan: | |
| options.variants = "default" | |
| if options.j == 0: | |
| options.j = multiprocessing.cpu_count() | |
| if options.random_seed_stress_count <= 1 and options.random_seed == 0: | |
| options.random_seed = self._random_seed() | |
| # Use developer defaults if no variant was specified. | |
| options.variants = options.variants or "dev" | |
| if options.variants == "infra_staging": | |
| options.variants = "exhaustive" | |
| options.infra_staging = True | |
| # Resolve variant aliases and dedupe. | |
| # TODO(machenbach): Don't mutate global variable. Rather pass mutated | |
| # version as local variable. | |
| VARIANTS = list(set(reduce( | |
| list.__add__, | |
| (VARIANT_ALIASES.get(v, [v]) for v in options.variants.split(",")), | |
| [], | |
| ))) | |
| if not set(VARIANTS).issubset(ALL_VARIANTS): | |
| print "All variants must be in %s" % str(ALL_VARIANTS) | |
| raise base_runner.TestRunnerError() | |
| def CheckTestMode(name, option): # pragma: no cover | |
| if not option in ["run", "skip", "dontcare"]: | |
| print "Unknown %s mode %s" % (name, option) | |
| raise base_runner.TestRunnerError() | |
| CheckTestMode("slow test", options.slow_tests) | |
| CheckTestMode("pass|fail test", options.pass_fail_tests) | |
| if self.build_config.no_i18n: | |
| base_runner.TEST_MAP["bot_default"].remove("intl") | |
| base_runner.TEST_MAP["default"].remove("intl") | |
| # TODO(machenbach): uncomment after infra side lands. | |
| # base_runner.TEST_MAP["d8_default"].remove("intl") | |
| def _setup_env(self): | |
| super(StandardTestRunner, self)._setup_env() | |
| symbolizer_option = self._get_external_symbolizer_option() | |
| if self.sancov_dir: | |
| os.environ['ASAN_OPTIONS'] = ":".join([ | |
| 'coverage=1', | |
| 'coverage_dir=%s' % self.sancov_dir, | |
| symbolizer_option, | |
| "allow_user_segv_handler=1", | |
| ]) | |
| def _random_seed(self): | |
| seed = 0 | |
| while not seed: | |
| seed = random.SystemRandom().randint(-2147483648, 2147483647) | |
| return seed | |
| def _execute(self, args, options, suites): | |
| print(">>> Running tests for %s.%s" % (self.build_config.arch, | |
| self.mode_name)) | |
| # Populate context object. | |
| # Simulators are slow, therefore allow a longer timeout. | |
| if self.build_config.arch in SLOW_ARCHS: | |
| options.timeout *= 2 | |
| options.timeout *= self.mode_options.timeout_scalefactor | |
| if self.build_config.predictable: | |
| # Predictable mode is slower. | |
| options.timeout *= 2 | |
| ctx = context.Context(self.build_config.arch, | |
| self.mode_options.execution_mode, | |
| self.outdir, | |
| self.mode_options.flags, | |
| options.verbose, | |
| options.timeout, | |
| options.isolates, | |
| options.command_prefix, | |
| options.extra_flags, | |
| self.build_config.no_i18n, | |
| options.random_seed, | |
| options.no_sorting, | |
| options.rerun_failures_count, | |
| options.rerun_failures_max, | |
| options.no_harness, | |
| use_perf_data=not options.swarming, | |
| sancov_dir=self.sancov_dir, | |
| infra_staging=options.infra_staging) | |
| # TODO(all): Combine "simulator" and "simulator_run". | |
| # TODO(machenbach): In GN we can derive simulator run from | |
| # target_arch != v8_target_arch in the dumped build config. | |
| simulator_run = ( | |
| not options.dont_skip_simulator_slow_tests and | |
| self.build_config.arch in [ | |
| 'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc', | |
| 'ppc64', 's390', 's390x'] and | |
| bool(base_runner.ARCH_GUESS) and | |
| self.build_config.arch != base_runner.ARCH_GUESS) | |
| # Find available test suites and read test cases from them. | |
| variables = { | |
| "arch": self.build_config.arch, | |
| "asan": self.build_config.asan, | |
| "byteorder": sys.byteorder, | |
| "dcheck_always_on": self.build_config.dcheck_always_on, | |
| "deopt_fuzzer": False, | |
| "gc_fuzzer": False, | |
| "gc_stress": options.gc_stress, | |
| "gcov_coverage": self.build_config.gcov_coverage, | |
| "isolates": options.isolates, | |
| "mode": self.mode_options.status_mode, | |
| "msan": self.build_config.msan, | |
| "no_harness": options.no_harness, | |
| "no_i18n": self.build_config.no_i18n, | |
| "no_snap": self.build_config.no_snap, | |
| "novfp3": options.novfp3, | |
| "predictable": self.build_config.predictable, | |
| "simulator": utils.UseSimulator(self.build_config.arch), | |
| "simulator_run": simulator_run, | |
| "system": utils.GuessOS(), | |
| "tsan": self.build_config.tsan, | |
| "ubsan_vptr": self.build_config.ubsan_vptr, | |
| } | |
| progress_indicator = progress.IndicatorNotifier() | |
| progress_indicator.Register( | |
| progress.PROGRESS_INDICATORS[options.progress]()) | |
| if options.junitout: # pragma: no cover | |
| progress_indicator.Register(progress.JUnitTestProgressIndicator( | |
| options.junitout, options.junittestsuite)) | |
| if options.json_test_results: | |
| progress_indicator.Register(progress.JsonTestProgressIndicator( | |
| options.json_test_results, | |
| self.build_config.arch, | |
| self.mode_options.execution_mode, | |
| ctx.random_seed)) | |
| if options.flakiness_results: # pragma: no cover | |
| progress_indicator.Register(progress.FlakinessTestProgressIndicator( | |
| options.flakiness_results)) | |
| if options.infra_staging: | |
| for s in suites: | |
| s.ReadStatusFile(variables) | |
| s.ReadTestCases(ctx) | |
| return self._run_test_procs(suites, args, options, progress_indicator, | |
| ctx) | |
| all_tests = [] | |
| num_tests = 0 | |
| for s in suites: | |
| s.ReadStatusFile(variables) | |
| s.ReadTestCases(ctx) | |
| if len(args) > 0: | |
| s.FilterTestCasesByArgs(args) | |
| all_tests += s.tests | |
| # First filtering by status applying the generic rules (tests without | |
| # variants) | |
| if options.warn_unused: | |
| tests = [(t.name, t.variant) for t in s.tests] | |
| s.statusfile.warn_unused_rules(tests, check_variant_rules=False) | |
| s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests) | |
| if options.cat: | |
| verbose.PrintTestSource(s.tests) | |
| continue | |
| variant_gen = s.CreateLegacyVariantsGenerator(VARIANTS) | |
| variant_tests = [ t.create_variant(v, flags) | |
| for t in s.tests | |
| for v in variant_gen.FilterVariantsByTest(t) | |
| for flags in variant_gen.GetFlagSets(t, v) ] | |
| if options.random_seed_stress_count > 1: | |
| # Duplicate test for random seed stress mode. | |
| def iter_seed_flags(): | |
| for _ in range(0, options.random_seed_stress_count): | |
| # Use given random seed for all runs (set by default in | |
| # execution.py) or a new random seed if none is specified. | |
| if options.random_seed: | |
| yield [] | |
| else: | |
| yield ["--random-seed=%d" % self._random_seed()] | |
| s.tests = [ | |
| t.create_variant(t.variant, flags, 'seed-stress-%d' % n) | |
| for t in variant_tests | |
| for n, flags in enumerate(iter_seed_flags()) | |
| ] | |
| else: | |
| s.tests = variant_tests | |
| # Second filtering by status applying also the variant-dependent rules. | |
| if options.warn_unused: | |
| tests = [(t.name, t.variant) for t in s.tests] | |
| s.statusfile.warn_unused_rules(tests, check_variant_rules=True) | |
| s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests) | |
| s.tests = self._shard_tests(s.tests, options) | |
| for t in s.tests: | |
| t.cmd = t.get_command(ctx) | |
| num_tests += len(s.tests) | |
| if options.cat: | |
| return 0 # We're done here. | |
| if options.report: | |
| verbose.PrintReport(all_tests) | |
| # Run the tests. | |
| start_time = time.time() | |
| if self.build_config.predictable: | |
| outproc_factory = predictable.get_outproc | |
| else: | |
| outproc_factory = None | |
| runner = execution.Runner(suites, progress_indicator, ctx, | |
| outproc_factory) | |
| exit_code = runner.Run(options.j) | |
| overall_duration = time.time() - start_time | |
| if options.time: | |
| verbose.PrintTestDurations(suites, runner.outputs, overall_duration) | |
| if num_tests == 0: | |
| print("Warning: no tests were run!") | |
| if exit_code == 1 and options.json_test_results: | |
| print("Force exit code 0 after failures. Json test results file " | |
| "generated with failure information.") | |
| exit_code = 0 | |
| if self.sancov_dir: | |
| # If tests ran with sanitizer coverage, merge coverage files in the end. | |
| try: | |
| print "Merging sancov files." | |
| subprocess.check_call([ | |
| sys.executable, | |
| join(self.basedir, "tools", "sanitizers", "sancov_merger.py"), | |
| "--coverage-dir=%s" % self.sancov_dir]) | |
| except: | |
| print >> sys.stderr, "Error: Merging sancov files failed." | |
| exit_code = 1 | |
| return exit_code | |
| def _shard_tests(self, tests, options): | |
| shard_run, shard_count = self._get_shard_info(options) | |
| if shard_count < 2: | |
| return tests | |
| count = 0 | |
| shard = [] | |
| for test in tests: | |
| if count % shard_count == shard_run - 1: | |
| shard.append(test) | |
| count += 1 | |
| return shard | |
| def _run_test_procs(self, suites, args, options, progress_indicator, | |
| context): | |
| jobs = options.j | |
| print '>>> Running with test processors' | |
| loader = LoadProc() | |
| tests_counter = TestsCounter() | |
| results = ResultsTracker() | |
| indicators = progress_indicator.ToProgressIndicatorProcs() | |
| execproc = ExecutionProc(jobs, context) | |
| procs = [ | |
| loader, | |
| NameFilterProc(args) if args else None, | |
| StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), | |
| self._create_shard_proc(options), | |
| tests_counter, | |
| VariantProc(VARIANTS), | |
| StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), | |
| ] + indicators + [ | |
| results, | |
| self._create_rerun_proc(context), | |
| execproc, | |
| ] | |
| procs = filter(None, procs) | |
| for i in xrange(0, len(procs) - 1): | |
| procs[i].connect_to(procs[i + 1]) | |
| tests = [t for s in suites for t in s.tests] | |
| tests.sort(key=lambda t: t.is_slow, reverse=True) | |
| loader.setup() | |
| loader.load_tests(tests) | |
| print '>>> Running %d base tests' % tests_counter.total | |
| tests_counter.remove_from_chain() | |
| execproc.start() | |
| for indicator in indicators: | |
| indicator.finished() | |
| print '>>> %d tests ran' % results.total | |
| exit_code = 0 | |
| if results.failed: | |
| exit_code = 1 | |
| if results.remaining: | |
| exit_code = 2 | |
| if exit_code == 1 and options.json_test_results: | |
| print("Force exit code 0 after failures. Json test results file " | |
| "generated with failure information.") | |
| exit_code = 0 | |
| return exit_code | |
| def _create_rerun_proc(self, ctx): | |
| if not ctx.rerun_failures_count: | |
| return None | |
| return RerunProc(ctx.rerun_failures_count, | |
| ctx.rerun_failures_max) | |
| if __name__ == '__main__': | |
| sys.exit(StandardTestRunner().execute()) |