# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A command to download new baselines for NeedsRebaseline tests.

This command checks the list of tests with NeedsRebaseline expectations,
and downloads the latest baselines for those tests from the results archived
by the continuous builders.
"""

import logging
import optparse
import re
import sys
import time
import traceback
import urllib2

from webkitpy.common.net.buildbot import Build, current_build_link
from webkitpy.layout_tests.models.test_expectations import TestExpectations, BASELINE_SUFFIX_LIST
from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand


_log = logging.getLogger(__name__)


class AutoRebaseline(AbstractParallelRebaselineCommand):
    name = 'auto-rebaseline'
    help_text = 'Rebaselines any NeedsRebaseline lines in TestExpectations that have cycled through all the bots.'
    AUTO_REBASELINE_BRANCH_NAME = 'auto-rebaseline-temporary-branch'
    AUTO_REBASELINE_ALT_BRANCH_NAME = 'auto-rebaseline-alt-temporary-branch'

    # Rietveld uploader stinks. Limit the number of rebaselines in a given patch to keep upload from failing.
    # FIXME: http://crbug.com/263676 Obviously we should fix the uploader here.
    MAX_LINES_TO_REBASELINE = 200

    SECONDS_BEFORE_GIVING_UP = 300

    def __init__(self):
        super(AutoRebaseline, self).__init__(options=[
            # FIXME: Remove this option.
            self.no_optimize_option,
            # FIXME: Remove this option.
            self.results_directory_option,
            optparse.make_option('--auth-refresh-token-json', help='Rietveld auth refresh JSON token.'),
            optparse.make_option('--dry-run', action='store_true', default=False,
                                 help='Run without creating a temporary branch, committing locally, or uploading/landing '
                                 'changes to the remote repository.')
        ])
        self._blame_regex = re.compile(r'''
                ^(\S*)      # Commit hash
                [^(]* \(    # Whitespace and open parenthesis
                <           # Email address is surrounded by <>
                (
                    [^@]+   # Username preceding @
                    @
                    [^@>]+  # Domain terminated by @ or >, some lines have an additional @ fragment after the email.
                )
                .*?([^ ]*)  # Test file name
                \ \[        # Single space followed by opening [ for expectation specifier
                [^[]*$      # Prevents matching previous [ for version specifiers instead of expectation specifiers
            ''', re.VERBOSE)

    def bot_revision_data(self, git):
        revisions = []
        for builder_name in self._release_builders():
            result = self._tool.buildbot.fetch_results(Build(builder_name))
            if result.run_was_interrupted():
                _log.error("Can't rebaseline because the latest run on %s exited early.", result.builder_name())
                return []
            revisions.append({
                'builder': result.builder_name(),
                'revision': result.chromium_revision(git),
            })
        return revisions

    @staticmethod
    def _strip_comments(line):
        comment_index = line.find('#')
        if comment_index == -1:
            comment_index = len(line)
        return re.sub(r"\s+", ' ', line[:comment_index].strip())

    def tests_to_rebaseline(self, tool, min_revision, print_revisions):
        port = tool.port_factory.get()
        expectations_file_path = port.path_to_generic_test_expectations_file()

        tests = set()
        revision = None
        commit = None
        author = None
        bugs = set()
        has_any_needs_rebaseline_lines = False

        for line in tool.git().blame(expectations_file_path).split('\n'):
            line = self._strip_comments(line)
            if 'NeedsRebaseline' not in line:
                continue

            has_any_needs_rebaseline_lines = True

            parsed_line = self._blame_regex.match(line)
            if not parsed_line:
                # Deal gracefully with inability to parse blame info for a line in TestExpectations.
                # Parsing could fail if for example during local debugging the developer modifies
                # TestExpectations and does not commit.
                _log.info("Couldn't find blame info for expectations line, skipping [line=%s].", line)
                continue

            commit_hash = parsed_line.group(1)
            commit_position = tool.git().commit_position_from_git_commit(commit_hash)

            test = parsed_line.group(3)
            if print_revisions:
                _log.info('%s is waiting for r%s', test, commit_position)

            if not commit_position or commit_position > min_revision:
                continue

            if revision and commit_position != revision:
                continue

            if not revision:
                revision = commit_position
                commit = commit_hash
                author = parsed_line.group(2)

            bugs.update(re.findall(r"crbug\.com\/(\d+)", line))
            tests.add(test)

            if len(tests) >= self.MAX_LINES_TO_REBASELINE:
                _log.info('Too many tests to rebaseline in one patch. Doing the first %d.', self.MAX_LINES_TO_REBASELINE)
                break

        return tests, revision, commit, author, bugs, has_any_needs_rebaseline_lines

    def commit_message(self, author, revision, commit, bugs):
        message = 'Auto-rebaseline for r%s\n\n' % revision
        build_link = current_build_link(self._tool)
        if build_link:
            message += 'Build: %s\n\n' % build_link
        message += '%s\n\n' % self.link_to_patch(commit)
        if bugs:
            message += 'BUG=%s\n' % ','.join(bugs)
        message += 'TBR=%s\n' % author
        return message

    @staticmethod
    def link_to_patch(commit):
        return 'https://chromium.googlesource.com/chromium/src/+/' + commit

    def get_test_prefix_list(self, tests):
        test_prefix_list = {}
        lines_to_remove = {}

        for builder_name in self._release_builders():
            port_name = self._tool.builders.port_name_for_builder_name(builder_name)
            port = self._tool.port_factory.get(port_name)
            expectations = TestExpectations(port, include_overrides=True)
            for test in expectations.get_needs_rebaseline_failures():
                if test not in tests:
                    continue

                if test not in test_prefix_list:
                    lines_to_remove[test] = []
                    test_prefix_list[test] = {}
                lines_to_remove[test].append(builder_name)
                test_prefix_list[test][Build(builder_name)] = BASELINE_SUFFIX_LIST

        return test_prefix_list, lines_to_remove

    def _run_git_cl_command(self, options, command):
        subprocess_command = ['git', 'cl'] + command
        if options.verbose:
            subprocess_command.append('--verbose')
        if options.auth_refresh_token_json:
            subprocess_command.append('--auth-refresh-token-json')
            subprocess_command.append(options.auth_refresh_token_json)

        process = self._tool.executive.popen(subprocess_command, stdout=self._tool.executive.PIPE,
                                             stderr=self._tool.executive.STDOUT)
        last_output_time = time.time()

        # git cl sometimes completely hangs. Bail if we haven't gotten any output to stdout/stderr in a while.
        while process.poll() is None and time.time() < last_output_time + self.SECONDS_BEFORE_GIVING_UP:
            # FIXME: This doesn't make any sense. readline blocks, so all this code to
            # try and bail is useless. Instead, we should do the readline calls on a
            # subthread. Then the rest of this code would make sense.
            out = process.stdout.readline().rstrip('\n')
            if out:
                last_output_time = time.time()
                _log.info(out)

        if process.poll() is None:
            _log.error('Command hung: %s', subprocess_command)
            return False
        return True

    # FIXME: Move this somewhere more general.
    @staticmethod
    def tree_status():
        blink_tree_status_url = 'http://chromium-status.appspot.com/status'
        status = urllib2.urlopen(blink_tree_status_url).read().lower()
        if 'closed' in status or status == '0':
            return 'closed'
        elif 'open' in status or status == '1':
            return 'open'
        return 'unknown'

    def execute(self, options, args, tool):
        self._tool = tool
        if tool.git().executable_name == 'svn':
            _log.error('Auto rebaseline only works with a git checkout.')
            return

        if not options.dry_run and tool.git().has_working_directory_changes():
            _log.error('Cannot proceed with working directory changes. Clean working directory first.')
            return

        revision_data = self.bot_revision_data(tool.git())
        if not revision_data:
            return

        min_revision = int(min([item['revision'] for item in revision_data]))
        tests, revision, commit, author, bugs, _ = self.tests_to_rebaseline(
            tool, min_revision, print_revisions=options.verbose)

        if options.verbose:
            _log.info('Min revision across all bots is %s.', min_revision)
            for item in revision_data:
                _log.info('%s: r%s', item['builder'], item['revision'])

        if not tests:
            _log.debug('No tests to rebaseline.')
            return

        if self.tree_status() == 'closed':
            _log.info('Cannot proceed. Tree is closed.')
            return

        _log.info('Rebaselining %s for r%s by %s.', list(tests), revision, author)

        test_prefix_list, _ = self.get_test_prefix_list(tests)

        did_switch_branches = False
        did_finish = False
        old_branch_name_or_ref = ''
        rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME
        try:
            # Save the current branch name and check out a clean branch for the patch.
            old_branch_name_or_ref = tool.git().current_branch_or_ref()
            if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME:
                rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME
            if not options.dry_run:
                tool.git().delete_branch(rebaseline_branch_name)
                tool.git().create_clean_branch(rebaseline_branch_name)
                did_switch_branches = True

            if test_prefix_list:
                self.rebaseline(options, test_prefix_list)

            if options.dry_run:
                return

            tool.git().commit_locally_with_message(
                self.commit_message(author, revision, commit, bugs))

            # FIXME: It would be nice if we could dcommit the patch without uploading, but still
            # go through all the precommit hooks. For rebaselines with lots of files, uploading
            # takes a long time and sometimes fails, but we don't want to commit if, e.g. the
            # tree is closed.
            did_finish = self._run_git_cl_command(options, ['upload', '-f'])

            if did_finish:
                # Uploading can take a very long time. Do another pull to make sure TestExpectations is up to date,
                # so the dcommit can go through.
                # FIXME: Log the pull and dcommit stdout/stderr to the log-server.
                tool.executive.run_command(['git', 'pull'])

                self._run_git_cl_command(options, ['land', '-f', '-v'])
        except OSError:
            traceback.print_exc(file=sys.stderr)
        finally:
            if did_switch_branches:
                if did_finish:
                    # Close the issue if dcommit failed.
                    issue_already_closed = tool.executive.run_command(
                        ['git', 'config', 'branch.%s.rietveldissue' % rebaseline_branch_name],
                        return_exit_code=True)
                    if not issue_already_closed:
                        self._run_git_cl_command(options, ['set_close'])

                tool.git().ensure_cleanly_tracking_remote_master()
                if old_branch_name_or_ref:
                    tool.git().checkout_branch(old_branch_name_or_ref)
                tool.git().delete_branch(rebaseline_branch_name)