src/third_party/devtools/scripts/test/run_type_check.py

#!/usr/bin/env python
# Copyright (c) 2012 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import argparse
import os
from os import path
import re
import shutil
import subprocess
import sys
import tempfile

SCRIPTS_PATH = path.dirname(path.dirname(path.abspath(__file__)))
sys.path.append(SCRIPTS_PATH)

from build import dependency_preprocessor
from build import generate_protocol_externs
from build import modular_build
from build import special_case_namespaces

import devtools_paths
import utils

try:
    import simplejson as json
except ImportError:
    import json

is_cygwin = sys.platform == 'cygwin'


def popen(arguments):
    return subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)


def to_platform_path(filepath):
    if not is_cygwin:
        return filepath
    return re.sub(r'^/cygdrive/(\w)', '\\1:', filepath)


def to_platform_path_exact(filepath):
    if not is_cygwin:
        return filepath
    output, _ = popen(['cygpath', '-w', filepath]).communicate()
    # pylint: disable=E1103
    return output.strip().replace('\\', '\\\\')


DEVTOOLS_PATH = devtools_paths.devtools_root_path()
SCRIPTS_PATH = path.join(DEVTOOLS_PATH, 'scripts')
ROOT_PATH = devtools_paths.root_path()
BROWSER_PROTOCOL_PATH = devtools_paths.browser_protocol_path()
# TODO(dgozman): move these checks to v8.
JS_PROTOCOL_PATH = path.join(ROOT_PATH, 'v8', 'include', 'js_protocol.pdl')
DEVTOOLS_FRONTEND_PATH = path.join(DEVTOOLS_PATH, 'front_end')
GLOBAL_EXTERNS_FILE = to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'externs.js'))
DEFAULT_PROTOCOL_EXTERNS_FILE = path.join(DEVTOOLS_FRONTEND_PATH, 'protocol_externs.js')
RUNTIME_FILE = to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'Runtime.js'))
ROOT_MODULE_FILE = to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'root.js'))

CLOSURE_COMPILER_JAR = to_platform_path(path.join(SCRIPTS_PATH, 'closure', 'compiler.jar'))
CLOSURE_RUNNER_JAR = to_platform_path(path.join(SCRIPTS_PATH, 'closure', 'closure_runner', 'closure_runner.jar'))
JSDOC_VALIDATOR_JAR = to_platform_path(path.join(SCRIPTS_PATH, 'jsdoc_validator', 'jsdoc_validator.jar'))

TYPE_CHECKED_JSDOC_TAGS_LIST = ['param', 'return', 'type', 'enum']
TYPE_CHECKED_JSDOC_TAGS_OR = '|'.join(TYPE_CHECKED_JSDOC_TAGS_LIST)

# Basic regex for invalid JsDoc types: an object type name ([A-Z][_A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
INVALID_TYPE_REGEX = re.compile(r'@(?:' + TYPE_CHECKED_JSDOC_TAGS_OR +
                                r')\s*\{.*(?<![!?:._A-Za-z0-9])([A-Z][_A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}')
INVALID_TYPE_DESIGNATOR_REGEX = re.compile(r'@(?:' + TYPE_CHECKED_JSDOC_TAGS_OR + r')\s*.*(?<![{: ])([?!])=?\}')
INVALID_NON_OBJECT_TYPE_REGEX = re.compile(r'@(?:' + TYPE_CHECKED_JSDOC_TAGS_OR + r')\s*\{.*(![a-z]+)[^/]*\}')
ERROR_WARNING_REGEX = re.compile(r'WARNING|ERROR')
LOADED_CSS_REGEX = re.compile(r'(?:registerRequiredCSS|WebInspector\.View\.createStyleElement)\s*\(\s*"(.+)"\s*\)')

JAVA_BUILD_REGEX = re.compile(r'\w+ version "(\d+)\.(\d+)')

def log_error(message):
    print 'ERROR: ' + message


def error_excepthook(exctype, value, traceback):
    print 'ERROR:'
    sys.__excepthook__(exctype, value, traceback)


sys.excepthook = error_excepthook

APPLICATION_DESCRIPTORS = [
    'inspector',
    'toolbox',
    'integration_test_runner',
    'formatter_worker',
    'heap_snapshot_worker',
]

SKIPPED_NAMESPACES = {
    'Console',  # Closure uses Console as a namespace item so we cannot override it right now.
    'Gonzales',  # third party module defined in front_end/externs.js
    'Terminal',  # third party module defined in front_end/externs.js
}


def has_errors(output):
    return re.search(ERROR_WARNING_REGEX, output) is not None


class JSDocChecker:

    def __init__(self, descriptors, java_exec):
        self._error_found = False
        self._all_files = descriptors.all_compiled_files()
        self._java_exec = java_exec

    def check(self):
        print 'Verifying JSDoc comments...'
        self._verify_jsdoc()
        self._run_jsdoc_validator()
        return self._error_found

    def _run_jsdoc_validator(self):
        files = [to_platform_path(f) for f in self._all_files]
        file_list = tempfile.NamedTemporaryFile(mode='wt', delete=False)
        try:
            file_list.write('\n'.join(files))
        finally:
            file_list.close()
        proc = popen(self._java_exec + ['-jar', JSDOC_VALIDATOR_JAR, '--files-list-name', to_platform_path_exact(file_list.name)])
        (out, _) = proc.communicate()
        if out:
            print('JSDoc validator output:%s%s' % (os.linesep, out))
            self._error_found = True
        os.remove(file_list.name)

    def _verify_jsdoc(self):
        for full_file_name in self._all_files:
            line_index = 0
            with open(full_file_name, 'r') as sourceFile:
                for line in sourceFile:
                    line_index += 1
                    if line.rstrip():
                        self._verify_jsdoc_line(full_file_name, line_index, line)

    def _verify_jsdoc_line(self, file_name, line_index, line):

        def print_error(message, error_position):
            print '%s:%s: ERROR - %s%s%s%s%s%s' % (file_name, line_index, message, os.linesep, line, os.linesep,
                                                   ' ' * error_position + '^', os.linesep)

        known_css = {}
        match = re.search(INVALID_TYPE_REGEX, line)
        if match:
            print_error('Type "%s" nullability not marked explicitly with "?" (nullable) or "!" (non-nullable)' % match.group(1),
                        match.start(1))
            self._error_found = True

        match = re.search(INVALID_NON_OBJECT_TYPE_REGEX, line)
        if match:
            print_error('Non-object type explicitly marked with "!" (non-nullable), which is the default and should be omitted',
                        match.start(1))
            self._error_found = True

        match = re.search(INVALID_TYPE_DESIGNATOR_REGEX, line)
        if match:
            print_error('Type nullability indicator misplaced, should precede type', match.start(1))
            self._error_found = True

        match = re.search(LOADED_CSS_REGEX, line)
        if match:
            css_file = path.join(DEVTOOLS_FRONTEND_PATH, match.group(1))
            exists = known_css.get(css_file)
            if exists is None:
                exists = path.isfile(css_file)
                known_css[css_file] = exists
            if not exists:
                print_error('Dynamically loaded CSS stylesheet is missing in the source tree', match.start(1))
                self._error_found = True


def find_java():
    # Based on http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python.
    def which(program):
        def is_executable(fpath):
            return path.isfile(fpath) and os.access(fpath, os.X_OK)

        fpath, fname = path.split(program)
        if fpath:
            if is_executable(program):
                return program
            return None
        env_paths = os.environ["PATH"].split(os.pathsep)
        if sys.platform == "win32":
            env_paths = get_windows_path(env_paths)
        for part in env_paths:
            part = part.strip('\"')
            file = path.join(part, program)
            if is_executable(file):
                return file
            if sys.platform == "win32" and not file.endswith(".exe"):
                file_exe = file + ".exe"
                if is_executable(file_exe):
                    return file_exe
        return None

    # Use to find 64-bit programs (e.g. Java) when using 32-bit python in Windows
    def get_windows_path(env_paths):
        new_env_paths = env_paths[:]
        for env_path in env_paths:
            env_path = env_path.lower()
            if "system32" in env_path:
                new_env_paths.append(env_path.replace("system32", "sysnative"))
        return new_env_paths

    required_major = 1
    required_minor = 7
    exec_command = None
    has_server_jvm = True
    java_path = which('java')

    if not java_path:
        print 'NOTE: No Java executable found in $PATH.'
        sys.exit(1)

    is_ok = False
    java_version_out, _ = popen([java_path, '-version']).communicate()
    # pylint: disable=E1103
    match = re.search(JAVA_BUILD_REGEX, java_version_out)
    if match:
        major = int(match.group(1))
        minor = int(match.group(2))
        is_ok = major > required_major or major == required_major and minor >= required_minor
    if is_ok:
        exec_command = [java_path, '-Xms1024m', '-server', '-XX:+TieredCompilation']
        check_server_proc = popen(exec_command + ['-version'])
        check_server_proc.communicate()
        if check_server_proc.returncode != 0:
            # Not all Java installs have server JVMs.
            exec_command = exec_command.remove('-server')
            has_server_jvm = False

    if not is_ok:
        print 'NOTE: Java executable version %d.%d or above not found in $PATH.' % (required_major, required_minor)
        sys.exit(1)
    print 'Java executable: %s%s' % (java_path, '' if has_server_jvm else ' (no server JVM)')
    return exec_command


common_closure_args = [
    '--summary_detail_level',
    '3',
    '--jscomp_error',
    'visibility',
    '--jscomp_warning',
    'missingOverride',
    '--compilation_level',
    'SIMPLE_OPTIMIZATIONS',
    '--warning_level',
    'VERBOSE',
    '--language_in=ECMASCRIPT_NEXT',
    '--language_out=ES5_STRICT',
    '--extra_annotation_name',
    'suppressReceiverCheck',
    '--extra_annotation_name',
    'suppressGlobalPropertiesCheck',
    '--checks-only',
]


def check_conditional_dependencies(modules_by_name):
    errors_found = False
    for name in modules_by_name:
        if 'test_runner' in name:
            continue
        for dep_name in modules_by_name[name].get('dependencies', []):
            dependency = modules_by_name[dep_name]
            if dependency.get('experiment') or dependency.get('condition'):
                log_error('Module "%s" may not depend on the conditional module "%s"' % (name, dep_name))
                errors_found = True
    return errors_found


def prepare_closure_frontend_compile(temp_devtools_path, descriptors, namespace_externs_path, protocol_externs_file):
    temp_frontend_path = path.join(temp_devtools_path, 'front_end')
    checker = dependency_preprocessor.DependencyPreprocessor(descriptors, temp_frontend_path, DEVTOOLS_FRONTEND_PATH)
    checker.enforce_dependencies()

    command = common_closure_args + [
        '--externs',
        to_platform_path(GLOBAL_EXTERNS_FILE),
        '--externs',
        namespace_externs_path,
        '--js',
        RUNTIME_FILE,
        '--js',
        ROOT_MODULE_FILE,
    ]

    all_files = descriptors.all_compiled_files()
    args = []
    for file in all_files:
        args.extend(['--js', file])
        if 'InspectorBackend.js' in file:
            args.extend(['--js', protocol_externs_file])

    for file in descriptors.all_skipped_compilation_files():
        # Write a dummy file for skipped compilation files that are autogenerated.
        # We don't type-check this file, but we import them via ES modules
        generated_file = path.join(temp_frontend_path, file)
        modular_build.write_file(generated_file, '')
        args.extend(['--js', generated_file])

    command += args
    command = [arg.replace(DEVTOOLS_FRONTEND_PATH, temp_frontend_path) for arg in command]
    compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
    try:
        compiler_args_file.write('devtools_frontend %s' % (' '.join(command)))
    finally:
        compiler_args_file.close()
    return compiler_args_file.name


def generate_namespace_externs(modules_by_name):

    def map_module_to_namespace(module):
        return special_case_namespaces.special_case_namespaces.get(module, to_camel_case(module))

    def to_camel_case(snake_string):
        components = snake_string.split('_')
        return ''.join(x.title() for x in components)

    all_namespaces = [map_module_to_namespace(module) for module in modules_by_name]
    namespaces = [namespace for namespace in all_namespaces if namespace not in SKIPPED_NAMESPACES]
    namespaces.sort()
    namespace_externs_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
    try:
        namespace_externs_file.write('var Root = {};\n')
        for namespace in namespaces:
            namespace_externs_file.write('var %s = {};\n' % namespace)
    finally:
        namespace_externs_file.close()
    namespace_externs_path = to_platform_path(namespace_externs_file.name)
    return namespace_externs_path


def main():
    protocol_externs_file = DEFAULT_PROTOCOL_EXTERNS_FILE
    errors_found = False
    parser = argparse.ArgumentParser()
    parser.add_argument('--protocol-externs-file')
    args, _ = parser.parse_known_args()
    if args.protocol_externs_file:
        protocol_externs_file = args.protocol_externs_file
    else:
        generate_protocol_externs.generate_protocol_externs(protocol_externs_file, BROWSER_PROTOCOL_PATH, JS_PROTOCOL_PATH)
    loader = modular_build.DescriptorLoader(DEVTOOLS_FRONTEND_PATH)
    descriptors = loader.load_applications(APPLICATION_DESCRIPTORS)
    modules_by_name = descriptors.modules

    java_exec = find_java()
    errors_found |= check_conditional_dependencies(modules_by_name)

    print 'Compiling frontend...'
    temp_devtools_path = tempfile.mkdtemp()
    namespace_externs_path = generate_namespace_externs(modules_by_name)
    compiler_args_file_path = prepare_closure_frontend_compile(temp_devtools_path, descriptors, namespace_externs_path,
                                                               protocol_externs_file)
    frontend_compile_proc = popen(
        java_exec + ['-jar', CLOSURE_RUNNER_JAR, '--compiler-args-file',
                     to_platform_path_exact(compiler_args_file_path)])

    print 'Compiling devtools_compatibility.js...'

    closure_compiler_command = java_exec + ['-jar', CLOSURE_COMPILER_JAR] + common_closure_args

    devtools_js_compile_command = closure_compiler_command + [
        '--externs',
        to_platform_path(GLOBAL_EXTERNS_FILE), '--jscomp_off=externsValidation', '--js',
        to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'devtools_compatibility.js'))
    ]
    devtools_js_compile_proc = popen(devtools_js_compile_command)

    errors_found |= JSDocChecker(descriptors, java_exec).check()

    (devtools_js_compile_out, _) = devtools_js_compile_proc.communicate()
    print 'devtools_compatibility.js compilation output:%s' % os.linesep, devtools_js_compile_out
    errors_found |= has_errors(devtools_js_compile_out)

    (frontend_compile_out, _) = frontend_compile_proc.communicate()
    print 'devtools frontend compilation output:'
    for line in frontend_compile_out.splitlines():
        if '@@ START_MODULE' in line or '@@ END_MODULE' in line:
            continue
        print line
    errors_found |= has_errors(frontend_compile_out)

    os.remove(protocol_externs_file)
    os.remove(namespace_externs_path)
    os.remove(compiler_args_file_path)
    shutil.rmtree(temp_devtools_path, True)

    if errors_found:
        print 'ERRORS DETECTED'
        sys.exit(1)
    print 'DONE - compiled without errors'


if __name__ == '__main__':
    main()