# Copyright (C) 2010 Google Inc. All rights reserved.
# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
# Copyright (C) 2011 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import logging
import optparse
import sys
import traceback

from webkitpy.common.host import Host
from webkitpy.layout_tests.controllers.manager import Manager
from webkitpy.layout_tests.generate_results_dashboard import DashBoardGenerator
from webkitpy.layout_tests.models import test_run_results
from webkitpy.layout_tests.port.factory import configuration_options, platform_options
from webkitpy.layout_tests.views import buildbot_results
from webkitpy.layout_tests.views import printing

_log = logging.getLogger(__name__)


def main(argv, stdout, stderr):
    options, args = parse_args(argv)

    if options.platform and 'test' in options.platform and not 'browser_test' in options.platform:
        # It's a bit lame to import mocks into real code, but this allows the user
        # to run tests against the test platform interactively, which is useful for
        # debugging test failures.
        from webkitpy.common.host_mock import MockHost
        host = MockHost()
    else:
        host = Host()

    try:
        port = host.port_factory.get(options.platform, options)
    except (NotImplementedError, ValueError) as error:
        # FIXME: is this the best way to handle unsupported port names?
        print >> stderr, str(error)
        return test_run_results.UNEXPECTED_ERROR_EXIT_STATUS

    try:
        return run(port, options, args, stderr, stdout).exit_code

    # We need to still handle KeyboardInterrupt, at least for webkitpy unittest cases.
    except KeyboardInterrupt:
        return test_run_results.INTERRUPTED_EXIT_STATUS
    except test_run_results.TestRunException as error:
        print >> stderr, error.msg
        return error.code
    except BaseException as error:
        if isinstance(error, Exception):
            print >> stderr, '\n%s raised: %s' % (error.__class__.__name__, error)
            traceback.print_exc(file=stderr)
        return test_run_results.UNEXPECTED_ERROR_EXIT_STATUS


def parse_args(args):
    option_group_definitions = []

    option_group_definitions.append(
        ('Platform options', platform_options()))

    option_group_definitions.append(
        ('Configuration options', configuration_options()))

    option_group_definitions.append(
        ('Printing Options', printing.print_options()))

    option_group_definitions.append(
        ('Android-specific Options', [
            optparse.make_option(
                '--adb-device',
                action='append',
                default=[],
                dest='adb_devices',
                help='Run Android layout tests on these devices.'),
            # FIXME: Flip this to be off by default once we can log the
            # device setup more cleanly.
            optparse.make_option(
                '--no-android-logging',
                dest='android_logging',
                action='store_false',
                default=True,
                help=('Do not log android-specific debug messages (default is to log as part '
                      'of --debug-rwt-logging')),
        ]))

    option_group_definitions.append(
        ('Results Options', [
            optparse.make_option(
                '--add-platform-exceptions',
                action='store_true',
                default=False,
                help=('Save generated results into the *most-specific-platform* directory rather '
                      'than the *generic-platform* directory')),
            optparse.make_option(
                '--additional-driver-flag',
                '--additional-drt-flag',
                dest='additional_driver_flag',
                action='append',
                default=[],
                help=('Additional command line flag to pass to the driver. Specify multiple '
                      'times to add multiple flags.')),
            optparse.make_option(
                '--additional-expectations',
                action='append',
                default=[],
                help=('Path to a test_expectations file that will override previous '
                      'expectations. Specify multiple times for multiple sets of overrides.')),
            optparse.make_option(
                '--additional-platform-directory',
                action='append',
                default=[],
                help=('Additional directory where to look for test baselines (will take '
                      'precedence over platform baselines). Specify multiple times to add '
                      'multiple search path entries.')),
            optparse.make_option(
                '--build-directory',
                help=('Path to the directory under which build files are kept (should not '
                      'include configuration)')),
            optparse.make_option(
                '--clobber-old-results',
                action='store_true',
                default=False,
                help='Clobbers test results from previous runs.'),
            optparse.make_option(
                '--compare-port',
                action='store',
                default=None,
                help="Use the specified port's baselines first"),
            optparse.make_option(
                '--driver-name',
                type='string',
                help='Alternative driver binary to use'),
            optparse.make_option(
                '--full-results-html',
                action='store_true',
                default=False,
                help='Show all failures in results.html, rather than only regressions'),
            optparse.make_option(
                '--json-test-results',
                action='store',
                help='Path to write the JSON test results to.'),
            optparse.make_option(
                '--new-baseline',
                action='store_true',
                default=False,
                help=('Save generated results as new baselines into the *most-specific-platform* '
                      "directory, overwriting whatever's already there. Equivalent to "
                      '--reset-results --add-platform-exceptions')),
            optparse.make_option(
                '--new-test-results',
                action='store_true',
                default=False,
                help='Create new baselines when no expected results exist'),
            optparse.make_option(
                '--no-show-results',
                dest='show_results',
                action='store_false',
                default=True,
                help="Don't launch a browser with results after the tests are done"),
            optparse.make_option(
                '-p',
                '--pixel',
                '--pixel-tests',
                dest='pixel_tests',
                action='store_true',
                default=True,
                help='Enable pixel-to-pixel PNG comparisons (enabled by default)'),
            optparse.make_option(
                '--no-pixel',
                '--no-pixel-tests',
                dest='pixel_tests',
                action='store_false',
                default=True,
                help='Disable pixel-to-pixel PNG comparisons'),
            # FIXME: we should support a comma separated list with
            # --pixel-test-directory as well.
            optparse.make_option(
                '--pixel-test-directory',
                dest='pixel_test_directories',
                action='append',
                default=[],
                help=('A directory where it is allowed to execute tests as pixel tests. Specify '
                      'multiple times to add multiple directories. This option implies '
                      '--pixel-tests. If specified, only those tests will be executed as pixel '
                      'tests that are located in one of the' ' directories enumerated with the '
                      'option. Some ports may ignore this option while others can have a default '
                      'value that can be overridden here.')),
            optparse.make_option(
                '--reset-results',
                action='store_true',
                default=False,
                help='Reset expectations to the generated results in their existing location.'),
            optparse.make_option(
                '--results-directory',
                help='Location of test results'),
            optparse.make_option(
                '--skip-failing-tests',
                action='store_true',
                default=False,
                help=('Skip tests that are expected to fail. Note: When using this option, '
                      'you might miss new crashes in these tests.')),
            optparse.make_option(
                '--smoke',
                action='store_true',
                help='Run just the SmokeTests'),
            optparse.make_option(
                '--no-smoke',
                dest='smoke',
                action='store_false',
                help='Do not run just the SmokeTests'),
        ]))

    option_group_definitions.append(
        ('Testing Options', [
            optparse.make_option(
                '--additional-env-var',
                type='string',
                action='append',
                default=[],
                help=('Passes that environment variable to the tests '
                      '(--additional-env-var=NAME=VALUE)')),
            optparse.make_option(
                '--batch-size',
                type='int',
                default=None,
                help=('Run a the tests in batches (n), after every n tests, the driver is '
                      'relaunched.')),
            optparse.make_option(
                '--build',
                dest='build',
                action='store_true',
                default=True,
                help=('Check to ensure the build is up to date (default).')),
            optparse.make_option(
                '--no-build',
                dest='build',
                action='store_false',
                help="Don't check to see if the build is up to date."),
            optparse.make_option(
                '--child-processes',
                help='Number of drivers to run in parallel.'),
            optparse.make_option(
                '--disable-breakpad',
                action='store_true',
                help="Don't use breakpad to symbolize unexpected crashes."),
            optparse.make_option(
                '--driver-logging',
                action='store_true',
                help='Print detailed logging of the driver/content_shell'),
            optparse.make_option(
                '--enable-leak-detection',
                action='store_true',
                help='Enable the leak detection of DOM objects.'),
            optparse.make_option(
                '--enable-sanitizer',
                action='store_true',
                help='Only alert on sanitizer-related errors and crashes'),
            optparse.make_option(
                '--exit-after-n-crashes-or-timeouts',
                type='int',
                default=None,
                help='Exit after the first N crashes instead of running all tests'),
            optparse.make_option(
                '--exit-after-n-failures',
                type='int',
                default=None,
                help='Exit after the first N failures instead of running all tests'),
            optparse.make_option(
                '--ignore-builder-category',
                action='store',
                help=('The category of builders to use with the --ignore-flaky-tests option '
                      "('layout' or 'deps').")),
            optparse.make_option(
                '--ignore-flaky-tests',
                action='store',
                help=('Control whether tests that are flaky on the bots get ignored. '
                      "'very-flaky' == Ignore any tests that flaked more than once on the bot. "
                      "'maybe-flaky' == Ignore any tests that flaked once on the bot. "
                      "'unexpected' == Ignore any tests that had unexpected results on the bot.")),
            optparse.make_option(
                '--iterations',
                type='int',
                default=1,
                help='Number of times to run the set of tests (e.g. ABCABCABC)'),
            optparse.make_option(
                '--layout-tests-directory',
                help=('Path to a custom layout tests directory')),
            optparse.make_option(
                '--max-locked-shards',
                type='int',
                default=0,
                help='Set the maximum number of locked shards'),
            optparse.make_option(
                '--nocheck-sys-deps',
                action='store_true',
                default=False,
                help="Don't check the system dependencies (themes)"),
            optparse.make_option(
                '--order',
                action='store',
                default='random',
                help=('Determine the order in which the test cases will be run. '
                      "'none' == use the order in which the tests were listed "
                      'either in arguments or test list, '
                      "'random' == pseudo-random order (default). Seed can be specified "
                      'via --seed, otherwise it will default to the current unix timestamp. '
                      "'natural' == use the natural order")),
            optparse.make_option(
                '--profile',
                action='store_true',
                help='Output per-test profile information.'),
            optparse.make_option(
                '--profiler',
                action='store',
                help='Output per-test profile information, using the specified profiler.'),
            optparse.make_option(
                '--repeat-each',
                type='int',
                default=1,
                help='Number of times to run each test (e.g. AAABBBCCC)'),
            # TODO(joelo): Delete --retry-failures and --no-retry-failures as they
            # are redundant with --num-retries.
            optparse.make_option(
                '--retry-failures',
                action='store_true',
                help=('Re-try any tests that produce unexpected results. Default is to not retry '
                      'if an explicit list of tests is passed to run-webkit-tests.')),
            optparse.make_option(
                '--no-retry-failures',
                dest='retry_failures',
                action='store_false',
                help="Don't re-try any tests that produce unexpected results."),
            optparse.make_option(
                '--num-retries',
                type='int',
                default=3,
                help=('Number of times to retry failures, default is 3. Only relevant when '
                      'failure retries are enabled.')),
            optparse.make_option(
                '--total-shards',
                type=int,
                help=('Total number of shards being used for this test run. '
                      'Must be used with --shard-index. '
                      '(The user of this script is responsible for spawning '
                      'all of the shards.)')),
            optparse.make_option(
                '--shard-index',
                type=int,
                help=('Shard index [0..total_shards) of this test run. '
                      'Must be used with --total-shards.')),
            optparse.make_option(
                '--run-singly',
                action='store_true',
                default=False,
                help='DEPRECATED, same as --batch-size=1 --verbose'),
            optparse.make_option(
                '--seed',
                type='int',
                help=('Seed to use for random test order (default: %default). '
                      'Only applicable in combination with --order=random.')),
            optparse.make_option(
                '--skipped',
                action='store',
                default=None,
                help=('control how tests marked SKIP are run. '
                      "'default' == Skip tests unless explicitly listed on the command line, "
                      "'ignore' == Run them anyway, "
                      "'only' == only run the SKIP tests, "
                      "'always' == always skip, even if listed on the command line.")),
            optparse.make_option(
                '--fastest',
                action='store',
                type='float',
                help='Run the N% fastest tests as well as any tests listed on the command line'),
            optparse.make_option(
                '--test-list',
                action='append',
                metavar='FILE',
                help='read list of tests to run from file'),
            optparse.make_option(
                '--time-out-ms',
                help='Set the timeout for each test'),
            optparse.make_option(
                '--wrapper',
                help=('wrapper command to insert before invocations of the driver; option '
                      "is split on whitespace before running. (Example: --wrapper='valgrind "
                      "--smc-check=all')")),
            # FIXME: Display default number of child processes that will run.
            optparse.make_option(
                '-f', '--fully-parallel',
                action='store_true',
                help='run all tests in parallel'),
            optparse.make_option(
                '-i', '--ignore-tests',
                action='append',
                default=[],
                help='directories or test to ignore (may specify multiple times)'),
            optparse.make_option(
                '-n', '--dry-run',
                action='store_true',
                default=False,
                help='Do everything but actually run the tests or upload results.'),
        ]))

    # FIXME: Move these into json_results_generator.py.
    option_group_definitions.append(
        ('Result JSON Options', [
            optparse.make_option(
                '--build-name',
                default='DUMMY_BUILD_NAME',
                help='The name of the builder used in its path, e.g. webkit-rel.'),
            optparse.make_option(
                '--step-name',
                default='webkit_tests',
                help='The name of the step in a build running this script.'),
            optparse.make_option(
                '--build-number',
                default='DUMMY_BUILD_NUMBER',
                help='The build number of the builder running this script.'),
            optparse.make_option(
                '--builder-name',
                default='',
                help=('The name of the builder shown on the waterfall running this script '
                      'e.g. WebKit.')),
            optparse.make_option(
                '--master-name',
                help='The name of the buildbot master.'),
            optparse.make_option(
                '--test-results-server',
                default='',
                help='If specified, upload results json files to this appengine server.'),
            optparse.make_option(
                '--write-full-results-to',
                help=('If specified, copy full_results.json from the results dir to the '
                      'specified path.')),
        ]))

    option_parser = optparse.OptionParser()

    for group_name, group_options in option_group_definitions:
        option_group = optparse.OptionGroup(option_parser, group_name)
        option_group.add_options(group_options)
        option_parser.add_option_group(option_group)

    return option_parser.parse_args(args)


def _set_up_derived_options(port, options, args):
    """Sets the options values that depend on other options values."""
    if options.batch_size is None:
        options.batch_size = port.default_batch_size()

    if not options.child_processes:
        options.child_processes = port.host.environ.get('WEBKIT_TEST_CHILD_PROCESSES',
                                                        str(port.default_child_processes()))
    if not options.max_locked_shards:
        options.max_locked_shards = int(port.host.environ.get('WEBKIT_TEST_MAX_LOCKED_SHARDS',
                                                              str(port.default_max_locked_shards())))

    if not options.configuration:
        options.configuration = port.default_configuration()

    if not options.time_out_ms:
        options.time_out_ms = str(port.default_timeout_ms())

    options.slow_time_out_ms = str(5 * int(options.time_out_ms))

    if options.additional_platform_directory:
        additional_platform_directories = []
        for path in options.additional_platform_directory:
            additional_platform_directories.append(port.host.filesystem.abspath(path))
        options.additional_platform_directory = additional_platform_directories

    if options.new_baseline:
        options.reset_results = True
        options.add_platform_exceptions = True

    if options.pixel_test_directories:
        options.pixel_tests = True
        verified_dirs = set()
        pixel_test_directories = options.pixel_test_directories
        for directory in pixel_test_directories:
            # FIXME: we should support specifying the directories all the ways we support it for additional
            # arguments specifying which tests and directories to run. We should also move the logic for that
            # to Port.
            filesystem = port.host.filesystem
            if not filesystem.isdir(filesystem.join(port.layout_tests_dir(), directory)):
                _log.warning("'%s' was passed to --pixel-test-directories, which doesn't seem to be a directory", str(directory))
            else:
                verified_dirs.add(directory)

        options.pixel_test_directories = list(verified_dirs)

    if options.run_singly:
        options.batch_size = 1
        options.verbose = True

    if not args and not options.test_list and options.smoke is None:
        options.smoke = port.default_smoke_test_only()
    if options.smoke:
        if not args and not options.test_list and options.retry_failures is None:
            # Retry failures by default if we're doing just a smoke test (no additional tests).
            options.retry_failures = True

        if not options.test_list:
            options.test_list = []
        options.test_list.append(port.host.filesystem.join(port.layout_tests_dir(), 'SmokeTests'))
        if not options.skipped:
            options.skipped = 'always'

    if not options.skipped:
        options.skipped = 'default'

    if not options.total_shards and 'GTEST_TOTAL_SHARDS' in port.host.environ:
        options.total_shards = int(port.host.environ['GTEST_TOTAL_SHARDS'])
    if not options.shard_index and 'GTEST_SHARD_INDEX' in port.host.environ:
        options.shard_index = int(port.host.environ['GTEST_SHARD_INDEX'])

    if not options.seed:
        options.seed = port.host.time()


def _run_tests(port, options, args, printer):
    _set_up_derived_options(port, options, args)
    manager = Manager(port, options, printer)
    printer.print_config(port.results_directory())
    return manager.run(args)


def run(port, options, args, logging_stream, stdout):
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)

    printer = printing.Printer(port, options, logging_stream, logger=logger)
    try:
        run_details = _run_tests(port, options, args, printer)
        printer.flush()

        if (not options.dry_run and
                (run_details.exit_code not in test_run_results.ERROR_CODES or
                 run_details.exit_code == test_run_results.EARLY_EXIT_STATUS) and
                not run_details.initial_results.keyboard_interrupted):
            bot_printer = buildbot_results.BuildBotPrinter(stdout, options.debug_rwt_logging)
            bot_printer.print_results(run_details)
            stdout.flush()

            _log.debug('Generating dashboard...')
            gen_dash_board = DashBoardGenerator(port)
            gen_dash_board.generate()
            _log.debug('Dashboard generated.')

        _log.debug('')
        _log.debug('Testing completed, Exit status: %d', run_details.exit_code)

        # Temporary process dump for debugging windows timeout issues, see crbug.com/522396.
        _log.debug('')
        _log.debug('Process dump:')
        for process in port.host.executive.process_dump():
            _log.debug('\t%s', process)

        return run_details

    finally:
        printer.cleanup()

if __name__ == '__main__':
    exit_code = main(sys.argv[1:], sys.stdout, sys.stderr)
    sys.exit(exit_code)