# Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above
#    copyright notice, this list of conditions and the following
#    disclaimer.
# 2. Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials
#    provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.

"""Logic for converting and copying files from a W3C repo.

This module is responsible for modifying and copying a subset of the tests from
a local W3C repository source directory into a destination directory.
"""

import logging
import mimetypes
import re

from webkitpy.common.webkit_finder import WebKitFinder
from webkitpy.layout_tests.models.test_expectations import TestExpectationParser
from webkitpy.w3c.test_parser import TestParser
from webkitpy.w3c.test_converter import convert_for_webkit

_log = logging.getLogger(__name__)


class TestCopier(object):

    def __init__(self, host, source_repo_path, dest_dir_name='external'):
        """Initializes variables to prepare for copying and converting files.

        Args:
            host: An instance of Host.
            source_repo_path: Path to the local checkout of a
                web-platform-tests or csswg-test repository.
            dest_dir_name: The name of the directory under the layout tests
                directory where imported tests should be copied to.
                TODO(qyearsley): This can be made into a constant.
        """
        self.host = host

        assert self.host.filesystem.exists(source_repo_path)
        self.source_repo_path = source_repo_path
        self.dest_dir_name = dest_dir_name

        self.filesystem = self.host.filesystem
        self.webkit_finder = WebKitFinder(self.filesystem)
        self._webkit_root = self.webkit_finder.webkit_base()
        self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('LayoutTests')
        self.destination_directory = self.filesystem.normpath(
            self.filesystem.join(
                self.layout_tests_dir,
                dest_dir_name,
                self.filesystem.basename(self.source_repo_path)))
        self.import_in_place = (self.source_repo_path == self.destination_directory)
        self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
        self.is_wpt = self.filesystem.basename(self.source_repo_path) == 'wpt'

        self.import_list = []

        # This is just a FYI list of CSS properties that still need to be prefixed,
        # which may be output after importing.
        self._prefixed_properties = {}

    def do_import(self):
        _log.info('Importing %s into %s', self.source_repo_path, self.destination_directory)
        self.find_importable_tests()
        self.import_tests()

    def find_importable_tests(self):
        """Walks through the source directory to find what tests should be imported.

        This function sets self.import_list, which contains information about how many
        tests are being imported, and their source and destination paths.
        """
        paths_to_skip = self.find_paths_to_skip()

        for root, dirs, files in self.filesystem.walk(self.source_repo_path):
            cur_dir = root.replace(self.dir_above_repo + '/', '') + '/'
            _log.debug('Scanning %s...', cur_dir)
            total_tests = 0
            reftests = 0
            jstests = 0

            # Files in 'tools' are not for browser testing, so we skip them.
            # See: http://web-platform-tests.org/writing-tests/general-guidelines.html#tools
            dirs_to_skip = ('.git', 'test-plan', 'tools')

            # We copy all files in 'support', including HTML without metadata.
            # See: http://web-platform-tests.org/writing-tests/general-guidelines.html#support-files
            dirs_to_include = ('resources', 'support')

            if dirs:
                for name in dirs_to_skip:
                    if name in dirs:
                        dirs.remove(name)

                for path in paths_to_skip:
                    path_base = path.replace(self.dest_dir_name + '/', '')
                    path_base = path_base.replace(cur_dir, '')
                    path_full = self.filesystem.join(root, path_base)
                    if path_base in dirs:
                        _log.info('Skipping: %s', path_full)
                        dirs.remove(path_base)
                        if self.import_in_place:
                            self.filesystem.rmtree(path_full)

            copy_list = []

            for filename in files:
                path_full = self.filesystem.join(root, filename)
                path_base = path_full.replace(self.source_repo_path + '/', '')
                path_base = self.destination_directory.replace(self.layout_tests_dir + '/', '') + '/' + path_base
                if path_base in paths_to_skip:
                    if self.import_in_place:
                        _log.debug('Pruning: %s', path_base)
                        self.filesystem.remove(path_full)
                        continue
                    else:
                        continue
                # FIXME: This block should really be a separate function, but the early-continues make that difficult.

                # TODO(qyearsley): Remove the below block.
                if filename != '.gitignore' and (filename.startswith('.') or filename.endswith('.pl')):
                    _log.debug('Skipping: %s', path_full)
                    _log.debug('  Reason: Hidden files and perl scripts are not necessary.')
                    continue

                if filename == 'OWNERS' or filename == 'reftest.list':
                    # See http://crbug.com/584660 and http://crbug.com/582838.
                    _log.debug('Skipping: %s', path_full)
                    _log.debug('  Reason: This file may cause Chromium presubmit to fail.')
                    continue

                mimetype = mimetypes.guess_type(path_full)
                if ('html' not in str(mimetype[0]) and
                        'application/xhtml+xml' not in str(mimetype[0]) and
                        'application/xml' not in str(mimetype[0])):
                    copy_list.append({'src': path_full, 'dest': filename})
                    continue

                if self.filesystem.basename(root) in dirs_to_include:
                    copy_list.append({'src': path_full, 'dest': filename})
                    continue

                test_parser = TestParser(path_full, self.host)
                test_info = test_parser.analyze_test()
                if test_info is None:
                    copy_list.append({'src': path_full, 'dest': filename})
                    continue

                if 'reference' in test_info.keys():
                    ref_path_full = test_info['reference']
                    if not self.filesystem.exists(ref_path_full):
                        _log.warning('Skipping: %s', path_full)
                        _log.warning('  Reason: Ref file "%s" was not found.', ref_path_full)
                        continue

                    if not self.is_wpt:
                        # For csswg-test, we still need to add a ref file
                        # using WebKit naming conventions. See crbug.com/268729.
                        # FIXME: Remove this when csswg-test is merged into wpt.
                        test_basename = self.filesystem.basename(test_info['test'])
                        ref_file = self.filesystem.splitext(test_basename)[0] + '-expected'
                        ref_file += self.filesystem.splitext(ref_path_full)[1]
                        copy_list.append({
                            'src': test_info['reference'],
                            'dest': ref_file,
                            'reference_support_info': test_info['reference_support_info'],
                        })

                    reftests += 1
                    total_tests += 1
                    copy_list.append({'src': test_info['test'], 'dest': filename})

                elif 'jstest' in test_info.keys():
                    jstests += 1
                    total_tests += 1
                    copy_list.append({'src': path_full, 'dest': filename, 'is_jstest': True})

            if copy_list:
                # Only add this directory to the list if there's something to import
                self.import_list.append({'dirname': root, 'copy_list': copy_list,
                                         'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests})

    def find_paths_to_skip(self):
        paths_to_skip = set()
        port = self.host.port_factory.get()
        w3c_import_expectations_path = self.webkit_finder.path_from_webkit_base('LayoutTests', 'W3CImportExpectations')
        w3c_import_expectations = self.filesystem.read_text_file(w3c_import_expectations_path)
        parser = TestExpectationParser(port, all_tests=(), is_lint_mode=False)
        expectation_lines = parser.parse(w3c_import_expectations_path, w3c_import_expectations)
        for line in expectation_lines:
            if 'SKIP' in line.expectations:
                if line.specifiers:
                    _log.warning('W3CImportExpectations:%s should not have any specifiers', line.line_numbers)
                    continue
                paths_to_skip.add(line.name)
        return paths_to_skip

    def import_tests(self):
        """Reads |self.import_list|, and converts and copies files to their destination."""
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy['total_tests']
            total_imported_reftests += dir_to_copy['reftests']
            total_imported_jstests += dir_to_copy['jstests']

            if not dir_to_copy['copy_list']:
                continue

            orig_path = dir_to_copy['dirname']

            relative_dir = self.filesystem.relpath(orig_path, self.source_repo_path)
            dest_dir = self.filesystem.join(self.destination_directory, relative_dir)

            if not self.filesystem.exists(dest_dir):
                self.filesystem.maybe_make_directory(dest_dir)

            copied_files = []

            for file_to_copy in dir_to_copy['copy_list']:
                copied_file = self.copy_file(file_to_copy, dest_dir)
                if copied_file:
                    copied_files.append(copied_file)

        _log.info('')
        _log.info('Import complete')
        _log.info('')
        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
        _log.info('Imported %d reftests', total_imported_reftests)
        _log.info('Imported %d JS tests', total_imported_jstests)
        _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
        _log.info('')

        if self._prefixed_properties:
            _log.info('Properties needing prefixes (by count):')
            for prefixed_property in sorted(self._prefixed_properties, key=lambda p: self._prefixed_properties[p]):
                _log.info('  %s: %s', prefixed_property, self._prefixed_properties[prefixed_property])

    def copy_file(self, file_to_copy, dest_dir):
        """Converts and copies a file, if it should be copied.

        Args:
            file_to_copy: A dict in a file copy list constructed by
                find_importable_tests, which represents one file to copy, including
                the keys:
                    "src": Absolute path to the source location of the file.
                    "destination": File name of the destination file.
                And possibly also the keys "reference_support_info" or "is_jstest".
            dest_dir: Path to the directory where the file should be copied.

        Returns:
            The path to the new file, relative to the Blink root (//third_party/WebKit).
        """
        source_path = self.filesystem.normpath(file_to_copy['src'])
        dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])

        if self.filesystem.isdir(source_path):
            _log.error('%s refers to a directory', source_path)
            return None

        if not self.filesystem.exists(source_path):
            _log.error('%s not found. Possible error in the test.', source_path)
            return None

        reference_support_info = file_to_copy.get('reference_support_info') or None

        if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
            if not self.import_in_place:
                self.filesystem.maybe_make_directory(self.filesystem.dirname(dest_path))

        relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
        # FIXME: Maybe doing a file diff is in order here for existing files?
        # In other words, there's no sense in overwriting identical files, but
        # there's no harm in copying the identical thing.
        _log.debug('  copying %s', relpath)

        if self.should_try_to_convert(file_to_copy, source_path, dest_dir):
            converted_file = convert_for_webkit(
                dest_dir, filename=source_path,
                reference_support_info=reference_support_info,
                host=self.host)
            for prefixed_property in converted_file[0]:
                self._prefixed_properties.setdefault(prefixed_property, 0)
                self._prefixed_properties[prefixed_property] += 1

            self.filesystem.write_text_file(dest_path, converted_file[1])
        else:
            if not self.import_in_place:
                self.filesystem.copyfile(source_path, dest_path)
                if self.filesystem.read_binary_file(source_path)[:2] == '#!':
                    self.filesystem.make_executable(dest_path)

        return dest_path.replace(self._webkit_root, '')

    @staticmethod
    def should_try_to_convert(file_to_copy, source_path, dest_dir):
        """Checks whether we should try to modify the file when importing."""
        if file_to_copy.get('is_jstest', False):
            return False

        # Conversion is not necessary for any tests in wpt now; see http://crbug.com/654081.
        # Note, we want to move away from converting files, see http://crbug.com/663773.
        if re.search(r'[/\\]external[/\\]wpt[/\\]', dest_dir):
            return False

        # Only HTML, XHTML and CSS files should be converted.
        mimetype, _ = mimetypes.guess_type(source_path)
        return mimetype in ('text/html', 'application/xhtml+xml', 'text/css')