blob: 84010c3f69e47f29f926d576440ddced9a69bbf3 [file] [log] [blame] [edit]
#!/usr/bin/env vpython
#
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Runs all types of tests from one unified interface."""
from __future__ import absolute_import
import argparse
import collections
import contextlib
import itertools
import logging
import os
import re
import shutil
import signal
import sys
import tempfile
import threading
import traceback
import unittest
# Import _strptime before threaded code. datetime.datetime.strptime is
# threadsafe except for the initial import of the _strptime module.
# See http://crbug.com/724524 and https://bugs.python.org/issue7980.
import _strptime # pylint: disable=unused-import
# pylint: disable=redefined-builtin
from six.moves import range # Needed for python 3 compatibility.
# pylint: disable=ungrouped-imports
from pylib.constants import host_paths
if host_paths.DEVIL_PATH not in sys.path:
sys.path.append(host_paths.DEVIL_PATH)
from devil import base_error
from devil.utils import reraiser_thread
from devil.utils import run_tests_helper
from pylib import constants
from pylib.base import base_test_result
from pylib.base import environment_factory
from pylib.base import output_manager
from pylib.base import output_manager_factory
from pylib.base import result_sink
from pylib.base import test_instance_factory
from pylib.base import test_run_factory
from pylib.results import json_results
from pylib.results import report_results
from pylib.results.presentation import test_results_presentation
from pylib.utils import local_utils
from pylib.utils import logdog_helper
from pylib.utils import logging_utils
from pylib.utils import test_filter
from py_utils import contextlib_ext
_DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join(
host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json'))
def _RealPath(arg):
if arg.startswith('//'):
arg = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT,
arg[2:].replace('/', os.sep)))
return os.path.realpath(arg)
def AddTestLauncherOptions(parser):
"""Adds arguments mirroring //base/test/launcher.
Args:
parser: The parser to which arguments should be added.
Returns:
The given parser.
"""
parser.add_argument(
'--test-launcher-retry-limit',
'--test_launcher_retry_limit',
'--num_retries', '--num-retries',
'--isolated-script-test-launcher-retry-limit',
dest='num_retries', type=int, default=2,
help='Number of retries for a test before '
'giving up (default: %(default)s).')
parser.add_argument(
'--test-launcher-summary-output',
'--json-results-file',
dest='json_results_file', type=os.path.realpath,
help='If set, will dump results in JSON form to the specified file. '
'Note that this will also trigger saving per-test logcats to '
'logdog.')
parser.add_argument(
'--test-launcher-shard-index',
type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0),
help='Index of the external shard to run.')
parser.add_argument(
'--test-launcher-total-shards',
type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
help='Total number of external shards.')
test_filter.AddFilterOptions(parser)
return parser
def AddCommandLineOptions(parser):
"""Adds arguments to support passing command-line flags to the device."""
parser.add_argument(
'--device-flags-file',
type=os.path.realpath,
help='The relative filepath to a file containing '
'command-line flags to set on the device')
parser.add_argument(
'--use-apk-under-test-flags-file',
action='store_true',
help='Wether to use the flags file for the apk under test. If set, '
"the filename will be looked up in the APK's PackageInfo.")
parser.set_defaults(allow_unknown=True)
parser.set_defaults(command_line_flags=None)
def AddTracingOptions(parser):
# TODO(shenghuazhang): Move this into AddCommonOptions once it's supported
# for all test types.
parser.add_argument(
'--trace-output',
metavar='FILENAME', type=os.path.realpath,
help='Path to save test_runner trace json output to.')
parser.add_argument(
'--trace-all',
action='store_true',
help='Whether to trace all function calls.')
def AddCommonOptions(parser):
"""Adds all common options to |parser|."""
default_build_type = os.environ.get('BUILDTYPE', 'Debug')
debug_or_release_group = parser.add_mutually_exclusive_group()
debug_or_release_group.add_argument(
'--debug',
action='store_const', const='Debug', dest='build_type',
default=default_build_type,
help='If set, run test suites under out/Debug. '
'Default is env var BUILDTYPE or Debug.')
debug_or_release_group.add_argument(
'--release',
action='store_const', const='Release', dest='build_type',
help='If set, run test suites under out/Release. '
'Default is env var BUILDTYPE or Debug.')
parser.add_argument(
'--break-on-failure', '--break_on_failure',
dest='break_on_failure', action='store_true',
help='Whether to break on failure.')
# TODO(jbudorick): Remove this once everything has switched to platform
# mode.
parser.add_argument(
'--enable-platform-mode',
action='store_true',
help='Run the test scripts in platform mode, which '
'conceptually separates the test runner from the '
'"device" (local or remote, real or emulated) on '
'which the tests are running. [experimental]')
parser.add_argument(
'-e', '--environment',
default='local', choices=constants.VALID_ENVIRONMENTS,
help='Test environment to run in (default: %(default)s).')
parser.add_argument(
'--local-output',
action='store_true',
help='Whether to archive test output locally and generate '
'a local results detail page.')
class FastLocalDevAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
namespace.enable_concurrent_adb = True
namespace.enable_device_cache = True
namespace.extract_test_list_from_filter = True
namespace.local_output = True
namespace.num_retries = 0
namespace.skip_clear_data = True
parser.add_argument(
'--fast-local-dev',
type=bool,
nargs=0,
action=FastLocalDevAction,
help='Alias for: --num-retries=0 --enable-device-cache '
'--enable-concurrent-adb --skip-clear-data '
'--extract-test-list-from-filter --local-output')
# TODO(jbudorick): Remove this once downstream bots have switched to
# api.test_results.
parser.add_argument(
'--flakiness-dashboard-server',
dest='flakiness_dashboard_server',
help=argparse.SUPPRESS)
parser.add_argument(
'--gs-results-bucket',
help='Google Storage bucket to upload results to.')
parser.add_argument(
'--output-directory',
dest='output_directory', type=os.path.realpath,
help='Path to the directory in which build files are'
' located (must include build type). This will take'
' precedence over --debug and --release')
parser.add_argument(
'-v', '--verbose',
dest='verbose_count', default=0, action='count',
help='Verbose level (multiple times for more)')
parser.add_argument(
'--repeat', '--gtest_repeat', '--gtest-repeat',
'--isolated-script-test-repeat',
dest='repeat', type=int, default=0,
help='Number of times to repeat the specified set of tests.')
# This is currently only implemented for gtests and instrumentation tests.
parser.add_argument(
'--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests',
'--isolated-script-test-also-run-disabled-tests',
dest='run_disabled', action='store_true',
help='Also run disabled tests if applicable.')
# These are currently only implemented for gtests.
parser.add_argument('--isolated-script-test-output',
help='If present, store test results on this path.')
parser.add_argument('--isolated-script-test-perf-output',
help='If present, store chartjson results on this path.')
AddTestLauncherOptions(parser)
def ProcessCommonOptions(args):
"""Processes and handles all common options."""
run_tests_helper.SetLogLevel(args.verbose_count, add_handler=False)
# pylint: disable=redefined-variable-type
if args.verbose_count > 0:
handler = logging_utils.ColorStreamHandler()
else:
handler = logging.StreamHandler(sys.stdout)
# pylint: enable=redefined-variable-type
handler.setFormatter(run_tests_helper.CustomFormatter())
logging.getLogger().addHandler(handler)
constants.SetBuildType(args.build_type)
if args.output_directory:
constants.SetOutputDirectory(args.output_directory)
def AddDeviceOptions(parser):
"""Adds device options to |parser|."""
parser = parser.add_argument_group('device arguments')
parser.add_argument(
'--adb-path',
type=os.path.realpath,
help='Specify the absolute path of the adb binary that '
'should be used.')
parser.add_argument('--denylist-file',
type=os.path.realpath,
help='Device denylist file.')
parser.add_argument(
'-d', '--device', nargs='+',
dest='test_devices',
help='Target device(s) for the test suite to run on.')
parser.add_argument(
'--enable-concurrent-adb',
action='store_true',
help='Run multiple adb commands at the same time, even '
'for the same device.')
parser.add_argument(
'--enable-device-cache',
action='store_true',
help='Cache device state to disk between runs')
parser.add_argument(
'--skip-clear-data',
action='store_true',
help='Do not wipe app data between tests. Use this to '
'speed up local development and never on bots '
'(increases flakiness)')
parser.add_argument(
'--recover-devices',
action='store_true',
help='Attempt to recover devices prior to the final retry. Warning: '
'this will cause all devices to reboot.')
parser.add_argument(
'--tool',
dest='tool',
help='Run the test under a tool '
'(use --tool help to list them)')
parser.add_argument(
'--upload-logcats-file',
action='store_true',
dest='upload_logcats_file',
help='Whether to upload logcat file to logdog.')
logcat_output_group = parser.add_mutually_exclusive_group()
logcat_output_group.add_argument(
'--logcat-output-dir', type=os.path.realpath,
help='If set, will dump logcats recorded during test run to directory. '
'File names will be the device ids with timestamps.')
logcat_output_group.add_argument(
'--logcat-output-file', type=os.path.realpath,
help='If set, will merge logcats recorded during test run and dump them '
'to the specified file.')
def AddEmulatorOptions(parser):
"""Adds emulator-specific options to |parser|."""
parser = parser.add_argument_group('emulator arguments')
parser.add_argument(
'--avd-config',
type=os.path.realpath,
help='Path to the avd config textpb. '
'(See //tools/android/avd/proto/ for message definition'
' and existing textpb files.)')
parser.add_argument(
'--emulator-count',
type=int,
default=1,
help='Number of emulators to use.')
parser.add_argument(
'--emulator-window',
action='store_true',
default=False,
help='Enable graphical window display on the emulator.')
def AddGTestOptions(parser):
"""Adds gtest options to |parser|."""
parser = parser.add_argument_group('gtest arguments')
parser.add_argument(
'--app-data-file',
action='append', dest='app_data_files',
help='A file path relative to the app data directory '
'that should be saved to the host.')
parser.add_argument(
'--app-data-file-dir',
help='Host directory to which app data files will be'
' saved. Used with --app-data-file.')
parser.add_argument(
'--delete-stale-data',
dest='delete_stale_data', action='store_true',
help='Delete stale test data on the device.')
parser.add_argument(
'--enable-xml-result-parsing',
action='store_true', help=argparse.SUPPRESS)
parser.add_argument(
'--executable-dist-dir',
type=os.path.realpath,
help="Path to executable's dist directory for native"
" (non-apk) tests.")
parser.add_argument(
'--extract-test-list-from-filter',
action='store_true',
help='When a test filter is specified, and the list of '
'tests can be determined from it, skip querying the '
'device for the list of all tests. Speeds up local '
'development, but is not safe to use on bots ('
'http://crbug.com/549214')
parser.add_argument(
'--gs-test-artifacts-bucket',
help=('If present, test artifacts will be uploaded to this Google '
'Storage bucket.'))
parser.add_argument(
'--render-test-output-dir',
help='If present, store rendering artifacts in this path.')
parser.add_argument(
'--runtime-deps-path',
dest='runtime_deps_path', type=os.path.realpath,
help='Runtime data dependency file from GN.')
parser.add_argument(
'-t', '--shard-timeout',
dest='shard_timeout', type=int, default=120,
help='Timeout to wait for each test (default: %(default)s).')
parser.add_argument(
'--store-tombstones',
dest='store_tombstones', action='store_true',
help='Add tombstones in results if crash.')
parser.add_argument(
'-s', '--suite',
dest='suite_name', nargs='+', metavar='SUITE_NAME', required=True,
help='Executable name of the test suite to run.')
parser.add_argument(
'--test-apk-incremental-install-json',
type=os.path.realpath,
help='Path to install json for the test apk.')
parser.add_argument('--test-launcher-batch-limit',
dest='test_launcher_batch_limit',
type=int,
help='The max number of tests to run in a shard. '
'Ignores non-positive ints and those greater than '
'MAX_SHARDS')
parser.add_argument(
'-w', '--wait-for-java-debugger', action='store_true',
help='Wait for java debugger to attach before running any application '
'code. Also disables test timeouts and sets retries=0.')
parser.add_argument(
'--coverage-dir',
type=os.path.realpath,
help='Directory in which to place all generated coverage files.')
def AddInstrumentationTestOptions(parser):
"""Adds Instrumentation test options to |parser|."""
parser = parser.add_argument_group('instrumentation arguments')
parser.add_argument(
'--additional-apk',
action='append', dest='additional_apks', default=[],
type=_RealPath,
help='Additional apk that must be installed on '
'the device when the tests are run')
parser.add_argument(
'-A', '--annotation',
dest='annotation_str',
help='Comma-separated list of annotations. Run only tests with any of '
'the given annotations. An annotation can be either a key or a '
'key-values pair. A test that has no annotation is considered '
'"SmallTest".')
# TODO(jbudorick): Remove support for name-style APK specification once
# bots are no longer doing it.
parser.add_argument(
'--apk-under-test',
help='Path or name of the apk under test.')
parser.add_argument(
'--module',
action='append',
dest='modules',
help='Specify Android App Bundle modules to install in addition to the '
'base module.')
parser.add_argument(
'--fake-module',
action='append',
dest='fake_modules',
help='Specify Android App Bundle modules to fake install in addition to '
'the real modules.')
parser.add_argument(
'--additional-locale',
action='append',
dest='additional_locales',
help='Specify locales in addition to the device locale to install splits '
'for when --apk-under-test is an Android App Bundle.')
parser.add_argument(
'--coverage-dir',
type=os.path.realpath,
help='Directory in which to place all generated '
'Jacoco coverage files.')
parser.add_argument(
'--delete-stale-data',
action='store_true', dest='delete_stale_data',
help='Delete stale test data on the device.')
parser.add_argument(
'--disable-dalvik-asserts',
dest='set_asserts', action='store_false', default=True,
help='Removes the dalvik.vm.enableassertions property')
parser.add_argument(
'--enable-java-deobfuscation',
action='store_true',
help='Deobfuscate java stack traces in test output and logcat.')
parser.add_argument(
'-E', '--exclude-annotation',
dest='exclude_annotation_str',
help='Comma-separated list of annotations. Exclude tests with these '
'annotations.')
def package_replacement(arg):
split_arg = arg.split(',')
if len(split_arg) != 2:
raise argparse.ArgumentError(
arg,
'Expected two comma-separated strings for --replace-system-package, '
'received %d' % len(split_arg))
PackageReplacement = collections.namedtuple('PackageReplacement',
['package', 'replacement_apk'])
return PackageReplacement(package=split_arg[0],
replacement_apk=_RealPath(split_arg[1]))
parser.add_argument(
'--replace-system-package',
type=package_replacement, default=None,
help='Specifies a system package to replace with a given APK for the '
'duration of the test. Given as a comma-separated pair of strings, '
'the first element being the package and the second the path to the '
'replacement APK. Only supports replacing one package. Example: '
'--replace-system-package com.example.app,path/to/some.apk')
parser.add_argument(
'--remove-system-package',
default=[],
action='append',
dest='system_packages_to_remove',
help='Specifies a system package to remove before testing if it exists '
'on the system. WARNING: THIS WILL PERMANENTLY REMOVE THE SYSTEM APP. '
'Unlike --replace-system-package, the app will not be restored after '
'tests are finished.')
parser.add_argument(
'--use-webview-provider',
type=_RealPath, default=None,
help='Use this apk as the webview provider during test. '
'The original provider will be restored if possible, '
"on Nougat the provider can't be determined and so "
'the system will choose the default provider.')
parser.add_argument(
'--runtime-deps-path',
dest='runtime_deps_path', type=os.path.realpath,
help='Runtime data dependency file from GN.')
parser.add_argument(
'--screenshot-directory',
dest='screenshot_dir', type=os.path.realpath,
help='Capture screenshots of test failures')
parser.add_argument(
'--shared-prefs-file',
dest='shared_prefs_file', type=_RealPath,
help='The relative path to a file containing JSON list of shared '
'preference files to edit and how to do so. Example list: '
'[{'
' "package": "com.package.example",'
' "filename": "ExampleSettings.xml",'
' "set": {'
' "boolean_key_in_xml": true,'
' "string_key_in_xml": "string_value"'
' },'
' "remove": ['
' "key_in_xml_to_remove"'
' ]'
'}]')
parser.add_argument(
'--store-tombstones',
action='store_true', dest='store_tombstones',
help='Add tombstones in results if crash.')
parser.add_argument(
'--strict-mode',
dest='strict_mode', default='testing',
help='StrictMode command-line flag set on the device, '
'death/testing to kill the process, off to stop '
'checking, flash to flash only. (default: %(default)s)')
parser.add_argument(
'--test-apk',
required=True,
help='Path or name of the apk containing the tests.')
parser.add_argument(
'--test-jar',
help='Path of jar containing test java files.')
parser.add_argument(
'--timeout-scale',
type=float,
help='Factor by which timeouts should be scaled.')
parser.add_argument(
'-w', '--wait-for-java-debugger', action='store_true',
help='Wait for java debugger to attach before running any application '
'code. Also disables test timeouts and sets retries=0.')
# WPR record mode.
parser.add_argument('--wpr-enable-record',
action='store_true',
default=False,
help='If true, WPR server runs in record mode.'
'otherwise, runs in replay mode.')
# These arguments are suppressed from the help text because they should
# only ever be specified by an intermediate script.
parser.add_argument(
'--apk-under-test-incremental-install-json',
help=argparse.SUPPRESS)
parser.add_argument(
'--test-apk-incremental-install-json',
type=os.path.realpath,
help=argparse.SUPPRESS)
def AddSkiaGoldTestOptions(parser):
"""Adds Skia Gold test options to |parser|."""
parser = parser.add_argument_group("Skia Gold arguments")
parser.add_argument(
'--code-review-system',
help='A non-default code review system to pass to pass to Gold, if '
'applicable')
parser.add_argument(
'--git-revision', help='The git commit currently being tested.')
parser.add_argument(
'--gerrit-issue',
help='The Gerrit issue this test is being run on, if applicable.')
parser.add_argument(
'--gerrit-patchset',
help='The Gerrit patchset this test is being run on, if applicable.')
parser.add_argument(
'--buildbucket-id',
help='The Buildbucket build ID that this test was triggered from, if '
'applicable.')
local_group = parser.add_mutually_exclusive_group()
local_group.add_argument(
'--local-pixel-tests',
action='store_true',
default=None,
help='Specifies to run the Skia Gold pixel tests in local mode. When run '
'in local mode, uploading to Gold is disabled and traditional '
'generated/golden/diff images are output instead of triage links. '
'Running in local mode also implies --no-luci-auth. If both this '
'and --no-local-pixel-tests are left unset, the test harness will '
'attempt to detect whether it is running on a workstation or not '
'and set the options accordingly.')
local_group.add_argument(
'--no-local-pixel-tests',
action='store_false',
dest='local_pixel_tests',
help='Specifies to run the Skia Gold pixel tests in non-local (bot) '
'mode. When run in this mode, data is actually uploaded to Gold and '
'triage links are generated. If both this and --local-pixel-tests '
'are left unset, the test harness will attempt to detect whether '
'it is running on a workstation or not and set the options '
'accordingly.')
parser.add_argument(
'--no-luci-auth',
action='store_true',
default=False,
help="Don't use the serve account provided by LUCI for authentication "
'with Skia Gold, instead relying on gsutil to be pre-authenticated. '
'Meant for testing locally instead of on the bots.')
parser.add_argument(
'--bypass-skia-gold-functionality',
action='store_true',
default=False,
help='Bypass all interaction with Skia Gold, effectively disabling the '
'image comparison portion of any tests that use Gold. Only meant to be '
'used in case a Gold outage occurs and cannot be fixed quickly.')
def AddJUnitTestOptions(parser):
"""Adds junit test options to |parser|."""
parser = parser.add_argument_group('junit arguments')
parser.add_argument(
'--coverage-on-the-fly',
action='store_true',
help='Generate coverage data by Jacoco on-the-fly instrumentation.')
parser.add_argument(
'--coverage-dir', type=os.path.realpath,
help='Directory to store coverage info.')
parser.add_argument(
'--package-filter',
help='Filters tests by package.')
parser.add_argument(
'--runner-filter',
help='Filters tests by runner class. Must be fully qualified.')
parser.add_argument(
'--shards',
default=-1,
type=int,
help='Number of shards to run junit tests in parallel on. Only 1 shard '
'is supported when test-filter is specified. Values less than 1 will '
'use auto select.')
parser.add_argument(
'-s', '--test-suite', required=True,
help='JUnit test suite to run.')
debug_group = parser.add_mutually_exclusive_group()
debug_group.add_argument(
'-w', '--wait-for-java-debugger', action='store_const', const='8701',
dest='debug_socket', help='Alias for --debug-socket=8701')
debug_group.add_argument(
'--debug-socket',
help='Wait for java debugger to attach at specified socket address '
'before running any application code. Also disables test timeouts '
'and sets retries=0.')
# These arguments are for Android Robolectric tests.
parser.add_argument(
'--robolectric-runtime-deps-dir',
help='Path to runtime deps for Robolectric.')
parser.add_argument(
'--resource-apk',
required=True,
help='Path to .ap_ containing binary resources for Robolectric.')
def AddLinkerTestOptions(parser):
parser = parser.add_argument_group('linker arguments')
parser.add_argument(
'--test-apk',
type=os.path.realpath,
help='Path to the linker test APK.')
def AddMonkeyTestOptions(parser):
"""Adds monkey test options to |parser|."""
parser = parser.add_argument_group('monkey arguments')
parser.add_argument('--browser',
required=True,
choices=list(constants.PACKAGE_INFO.keys()),
metavar='BROWSER',
help='Browser under test.')
parser.add_argument(
'--category',
nargs='*', dest='categories', default=[],
help='A list of allowed categories. Monkey will only visit activities '
'that are listed with one of the specified categories.')
parser.add_argument(
'--event-count',
default=10000, type=int,
help='Number of events to generate (default: %(default)s).')
parser.add_argument(
'--seed',
type=int,
help='Seed value for pseudo-random generator. Same seed value generates '
'the same sequence of events. Seed is randomized by default.')
parser.add_argument(
'--throttle',
default=100, type=int,
help='Delay between events (ms) (default: %(default)s). ')
def AddPythonTestOptions(parser):
parser = parser.add_argument_group('python arguments')
parser.add_argument('-s',
'--suite',
dest='suite_name',
metavar='SUITE_NAME',
choices=list(constants.PYTHON_UNIT_TEST_SUITES.keys()),
help='Name of the test suite to run.')
def _CreateClassToFileNameDict(test_apk):
"""Creates a dict mapping classes to file names from size-info apk."""
constants.CheckOutputDirectory()
test_apk_size_info = os.path.join(constants.GetOutDirectory(), 'size-info',
os.path.basename(test_apk) + '.jar.info')
class_to_file_dict = {}
# Some tests such as webview_cts_tests use a separately downloaded apk to run
# tests. This means the apk may not have been built by the system and hence
# no size info file exists.
if not os.path.exists(test_apk_size_info):
logging.debug('Apk size file not found. %s', test_apk_size_info)
return class_to_file_dict
with open(test_apk_size_info, 'r') as f:
for line in f:
file_class, file_name = line.rstrip().split(',', 1)
# Only want files that are not prebuilt.
if file_name.startswith('../../'):
class_to_file_dict[file_class] = str(
file_name.replace('../../', '//', 1))
return class_to_file_dict
def _RunPythonTests(args):
"""Subcommand of RunTestsCommand which runs python unit tests."""
suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
suite_path = suite_vars['path']
suite_test_modules = suite_vars['test_modules']
sys.path = [suite_path] + sys.path
try:
suite = unittest.TestSuite()
suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
for m in suite_test_modules)
runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
return 0 if runner.run(suite).wasSuccessful() else 1
finally:
sys.path = sys.path[1:]
_DEFAULT_PLATFORM_MODE_TESTS = [
'gtest', 'instrumentation', 'junit', 'linker', 'monkey'
]
def RunTestsCommand(args, result_sink_client=None):
"""Checks test type and dispatches to the appropriate function.
Args:
args: argparse.Namespace object.
result_sink_client: A ResultSinkClient object.
Returns:
Integer indicated exit code.
Raises:
Exception: Unknown command name passed in, or an exception from an
individual test runner.
"""
command = args.command
ProcessCommonOptions(args)
logging.info('command: %s', ' '.join(sys.argv))
if args.enable_platform_mode or command in _DEFAULT_PLATFORM_MODE_TESTS:
return RunTestsInPlatformMode(args, result_sink_client)
if command == 'python':
return _RunPythonTests(args)
else:
raise Exception('Unknown test type.')
_SUPPORTED_IN_PLATFORM_MODE = [
# TODO(jbudorick): Add support for more test types.
'gtest',
'instrumentation',
'junit',
'linker',
'monkey',
]
def RunTestsInPlatformMode(args, result_sink_client=None):
def infra_error(message):
logging.fatal(message)
sys.exit(constants.INFRA_EXIT_CODE)
if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
infra_error('%s is not yet supported in platform mode' % args.command)
### Set up sigterm handler.
contexts_to_notify_on_sigterm = []
def unexpected_sigterm(_signum, _frame):
msg = [
'Received SIGTERM. Shutting down.',
]
for live_thread in threading.enumerate():
# pylint: disable=protected-access
thread_stack = ''.join(traceback.format_stack(
sys._current_frames()[live_thread.ident]))
msg.extend([
'Thread "%s" (ident: %s) is currently running:' % (
live_thread.name, live_thread.ident),
thread_stack])
for context in contexts_to_notify_on_sigterm:
context.ReceivedSigterm()
infra_error('\n'.join(msg))
signal.signal(signal.SIGTERM, unexpected_sigterm)
### Set up results handling.
# TODO(jbudorick): Rewrite results handling.
# all_raw_results is a list of lists of
# base_test_result.TestRunResults objects. Each instance of
# TestRunResults contains all test results produced by a single try,
# while each list of TestRunResults contains all tries in a single
# iteration.
all_raw_results = []
# all_iteration_results is a list of base_test_result.TestRunResults
# objects. Each instance of TestRunResults contains the last test
# result for each test run in that iteration.
all_iteration_results = []
global_results_tags = set()
json_file = tempfile.NamedTemporaryFile(delete=False)
json_file.close()
@contextlib.contextmanager
def json_finalizer():
try:
yield
finally:
if args.json_results_file and os.path.exists(json_file.name):
shutil.move(json_file.name, args.json_results_file)
elif args.isolated_script_test_output and os.path.exists(json_file.name):
shutil.move(json_file.name, args.isolated_script_test_output)
else:
os.remove(json_file.name)
@contextlib.contextmanager
def json_writer():
try:
yield
except Exception:
global_results_tags.add('UNRELIABLE_RESULTS')
raise
finally:
if args.isolated_script_test_output:
interrupted = 'UNRELIABLE_RESULTS' in global_results_tags
json_results.GenerateJsonTestResultFormatFile(all_raw_results,
interrupted,
json_file.name,
indent=2)
else:
json_results.GenerateJsonResultsFile(
all_raw_results,
json_file.name,
global_tags=list(global_results_tags),
indent=2)
test_class_to_file_name_dict = {}
# Test Location is only supported for instrumentation tests as it
# requires the size-info file.
if test_instance.TestType() == 'instrumentation':
test_class_to_file_name_dict = _CreateClassToFileNameDict(args.test_apk)
if result_sink_client:
for run in all_raw_results:
for results in run:
for r in results.GetAll():
# Matches chrome.page_info.PageInfoViewTest#testChromePage
match = re.search(r'^(.+\..+)#', r.GetName())
test_file_name = test_class_to_file_name_dict.get(
match.group(1)) if match else None
# Some tests put in non utf-8 char as part of the test
# which breaks uploads, so need to decode and re-encode.
result_sink_client.Post(
r.GetName(), r.GetType(), r.GetDuration(),
r.GetLog().decode('utf-8', 'replace').encode('utf-8'),
test_file_name)
@contextlib.contextmanager
def upload_logcats_file():
try:
yield
finally:
if not args.logcat_output_file:
logging.critical('Cannot upload logcat file: no file specified.')
elif not os.path.exists(args.logcat_output_file):
logging.critical("Cannot upload logcat file: file doesn't exist.")
else:
with open(args.logcat_output_file) as src:
dst = logdog_helper.open_text('unified_logcats')
if dst:
shutil.copyfileobj(src, dst)
dst.close()
logging.critical(
'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats'))
logcats_uploader = contextlib_ext.Optional(
upload_logcats_file(),
'upload_logcats_file' in args and args.upload_logcats_file)
### Set up test objects.
out_manager = output_manager_factory.CreateOutputManager(args)
env = environment_factory.CreateEnvironment(
args, out_manager, infra_error)
test_instance = test_instance_factory.CreateTestInstance(args, infra_error)
test_run = test_run_factory.CreateTestRun(env, test_instance, infra_error)
contexts_to_notify_on_sigterm.append(env)
contexts_to_notify_on_sigterm.append(test_run)
### Run.
with out_manager, json_finalizer():
with json_writer(), logcats_uploader, env, test_instance, test_run:
repetitions = (range(args.repeat +
1) if args.repeat >= 0 else itertools.count())
result_counts = collections.defaultdict(
lambda: collections.defaultdict(int))
iteration_count = 0
for _ in repetitions:
# raw_results will be populated with base_test_result.TestRunResults by
# test_run.RunTests(). It is immediately added to all_raw_results so
# that in the event of an exception, all_raw_results will already have
# the up-to-date results and those can be written to disk.
raw_results = []
all_raw_results.append(raw_results)
test_run.RunTests(raw_results)
if not raw_results:
all_raw_results.pop()
continue
iteration_results = base_test_result.TestRunResults()
for r in reversed(raw_results):
iteration_results.AddTestRunResults(r)
all_iteration_results.append(iteration_results)
iteration_count += 1
for r in iteration_results.GetAll():
result_counts[r.GetName()][r.GetType()] += 1
report_results.LogFull(
results=iteration_results,
test_type=test_instance.TestType(),
test_package=test_run.TestPackage(),
annotation=getattr(args, 'annotations', None),
flakiness_server=getattr(args, 'flakiness_dashboard_server',
None))
if args.break_on_failure and not iteration_results.DidRunPass():
break
if iteration_count > 1:
# display summary results
# only display results for a test if at least one test did not pass
all_pass = 0
tot_tests = 0
for test_name in result_counts:
tot_tests += 1
if any(result_counts[test_name][x] for x in (
base_test_result.ResultType.FAIL,
base_test_result.ResultType.CRASH,
base_test_result.ResultType.TIMEOUT,
base_test_result.ResultType.UNKNOWN)):
logging.critical(
'%s: %s',
test_name,
', '.join('%s %s' % (str(result_counts[test_name][i]), i)
for i in base_test_result.ResultType.GetTypes()))
else:
all_pass += 1
logging.critical('%s of %s tests passed in all %s runs',
str(all_pass),
str(tot_tests),
str(iteration_count))
if (args.local_output or not local_utils.IsOnSwarming()
) and not args.isolated_script_test_output:
with out_manager.ArchivedTempfile(
'test_results_presentation.html',
'test_results_presentation',
output_manager.Datatype.HTML) as results_detail_file:
result_html_string, _, _ = test_results_presentation.result_details(
json_path=json_file.name,
test_name=args.command,
cs_base_url='http://cs.chromium.org',
local_output=True)
results_detail_file.write(result_html_string.encode('utf-8'))
results_detail_file.flush()
logging.critical('TEST RESULTS: %s', results_detail_file.Link())
ui_screenshots = test_results_presentation.ui_screenshot_set(
json_file.name)
if ui_screenshots:
with out_manager.ArchivedTempfile(
'ui_screenshots.json',
'ui_capture',
output_manager.Datatype.JSON) as ui_screenshot_file:
ui_screenshot_file.write(ui_screenshots)
logging.critical('UI Screenshots: %s', ui_screenshot_file.Link())
return (0 if all(r.DidRunPass() for r in all_iteration_results)
else constants.ERROR_EXIT_CODE)
def DumpThreadStacks(_signal, _frame):
for thread in threading.enumerate():
reraiser_thread.LogThreadStack(thread)
def main():
signal.signal(signal.SIGUSR1, DumpThreadStacks)
parser = argparse.ArgumentParser()
command_parsers = parser.add_subparsers(
title='test types', dest='command')
subp = command_parsers.add_parser(
'gtest',
help='googletest-based C++ tests')
AddCommonOptions(subp)
AddDeviceOptions(subp)
AddEmulatorOptions(subp)
AddGTestOptions(subp)
AddTracingOptions(subp)
AddCommandLineOptions(subp)
subp = command_parsers.add_parser(
'instrumentation',
help='InstrumentationTestCase-based Java tests')
AddCommonOptions(subp)
AddDeviceOptions(subp)
AddEmulatorOptions(subp)
AddInstrumentationTestOptions(subp)
AddSkiaGoldTestOptions(subp)
AddTracingOptions(subp)
AddCommandLineOptions(subp)
subp = command_parsers.add_parser(
'junit',
help='JUnit4-based Java tests')
AddCommonOptions(subp)
AddJUnitTestOptions(subp)
subp = command_parsers.add_parser(
'linker',
help='linker tests')
AddCommonOptions(subp)
AddDeviceOptions(subp)
AddEmulatorOptions(subp)
AddLinkerTestOptions(subp)
subp = command_parsers.add_parser(
'monkey',
help="tests based on Android's monkey command")
AddCommonOptions(subp)
AddDeviceOptions(subp)
AddEmulatorOptions(subp)
AddMonkeyTestOptions(subp)
subp = command_parsers.add_parser(
'python',
help='python tests based on unittest.TestCase')
AddCommonOptions(subp)
AddPythonTestOptions(subp)
args, unknown_args = parser.parse_known_args()
if unknown_args:
if hasattr(args, 'allow_unknown') and args.allow_unknown:
args.command_line_flags = unknown_args
else:
parser.error('unrecognized arguments: %s' % ' '.join(unknown_args))
# --replace-system-package/--remove-system-package has the potential to cause
# issues if --enable-concurrent-adb is set, so disallow that combination.
concurrent_adb_enabled = (hasattr(args, 'enable_concurrent_adb')
and args.enable_concurrent_adb)
replacing_system_packages = (hasattr(args, 'replace_system_package')
and args.replace_system_package)
removing_system_packages = (hasattr(args, 'system_packages_to_remove')
and args.system_packages_to_remove)
if (concurrent_adb_enabled
and (replacing_system_packages or removing_system_packages)):
parser.error('--enable-concurrent-adb cannot be used with either '
'--replace-system-package or --remove-system-package')
# --use-webview-provider has the potential to cause issues if
# --enable-concurrent-adb is set, so disallow that combination
if (hasattr(args, 'use_webview_provider') and
hasattr(args, 'enable_concurrent_adb') and args.use_webview_provider and
args.enable_concurrent_adb):
parser.error('--use-webview-provider and --enable-concurrent-adb cannot '
'be used together')
if (getattr(args, 'coverage_on_the_fly', False)
and not getattr(args, 'coverage_dir', '')):
parser.error('--coverage-on-the-fly requires --coverage-dir')
if (hasattr(args, 'debug_socket') or
(hasattr(args, 'wait_for_java_debugger') and
args.wait_for_java_debugger)):
args.num_retries = 0
# Result-sink may not exist in the environment if rdb stream is not enabled.
result_sink_client = result_sink.TryInitClient()
try:
return RunTestsCommand(args, result_sink_client)
except base_error.BaseError as e:
logging.exception('Error occurred.')
if e.is_infra_error:
return constants.INFRA_EXIT_CODE
return constants.ERROR_EXIT_CODE
except: # pylint: disable=W0702
logging.exception('Unrecognized error occurred.')
return constants.ERROR_EXIT_CODE
if __name__ == '__main__':
sys.exit(main())