| #!/usr/bin/env python | |
| # | |
| # Copyright 2012 the V8 project authors. All rights reserved. | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions are | |
| # met: | |
| # | |
| # * Redistributions of source code must retain the above copyright | |
| # notice, this list of conditions and the following disclaimer. | |
| # * Redistributions in binary form must reproduce the above | |
| # copyright notice, this list of conditions and the following | |
| # disclaimer in the documentation and/or other materials provided | |
| # with the distribution. | |
| # * Neither the name of Google Inc. nor the names of its | |
| # contributors may be used to endorse or promote products derived | |
| # from this software without specific prior written permission. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| try: | |
| import hashlib | |
| md5er = hashlib.md5 | |
| except ImportError, e: | |
| import md5 | |
| md5er = md5.new | |
| import json | |
| import optparse | |
| import os | |
| from os.path import abspath, join, dirname, basename, exists | |
| import pickle | |
| import re | |
| import sys | |
| import subprocess | |
| import multiprocessing | |
| from subprocess import PIPE | |
| from testrunner.local import statusfile | |
| from testrunner.local import testsuite | |
| from testrunner.local import utils | |
| # Special LINT rules diverging from default and reason. | |
| # build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_". | |
| # build/include_what_you_use: Started giving false positives for variables | |
| # named "string" and "map" assuming that you needed to include STL headers. | |
| LINT_RULES = """ | |
| -build/header_guard | |
| -build/include_what_you_use | |
| -readability/fn_size | |
| -runtime/references | |
| """.split() | |
| LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing') | |
| FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n") | |
| ASSERT_OPTIMIZED_PATTERN = re.compile("assertOptimized") | |
| FLAGS_ENABLE_OPT = re.compile("//\s*Flags:.*--opt[^-].*\n") | |
| ASSERT_UNOPTIMIZED_PATTERN = re.compile("assertUnoptimized") | |
| FLAGS_NO_ALWAYS_OPT = re.compile("//\s*Flags:.*--no-?always-opt.*\n") | |
| TOOLS_PATH = dirname(abspath(__file__)) | |
| def CppLintWorker(command): | |
| try: | |
| process = subprocess.Popen(command, stderr=subprocess.PIPE) | |
| process.wait() | |
| out_lines = "" | |
| error_count = -1 | |
| while True: | |
| out_line = process.stderr.readline() | |
| if out_line == '' and process.poll() != None: | |
| if error_count == -1: | |
| print "Failed to process %s" % command.pop() | |
| return 1 | |
| break | |
| m = LINT_OUTPUT_PATTERN.match(out_line) | |
| if m: | |
| out_lines += out_line | |
| error_count += 1 | |
| sys.stdout.write(out_lines) | |
| return error_count | |
| except KeyboardInterrupt: | |
| process.kill() | |
| except: | |
| print('Error running cpplint.py. Please make sure you have depot_tools' + | |
| ' in your $PATH. Lint check skipped.') | |
| process.kill() | |
| class FileContentsCache(object): | |
| def __init__(self, sums_file_name): | |
| self.sums = {} | |
| self.sums_file_name = sums_file_name | |
| def Load(self): | |
| try: | |
| sums_file = None | |
| try: | |
| sums_file = open(self.sums_file_name, 'r') | |
| self.sums = pickle.load(sums_file) | |
| except: | |
| # Cannot parse pickle for any reason. Not much we can do about it. | |
| pass | |
| finally: | |
| if sums_file: | |
| sums_file.close() | |
| def Save(self): | |
| try: | |
| sums_file = open(self.sums_file_name, 'w') | |
| pickle.dump(self.sums, sums_file) | |
| except: | |
| # Failed to write pickle. Try to clean-up behind us. | |
| if sums_file: | |
| sums_file.close() | |
| try: | |
| os.unlink(self.sums_file_name) | |
| except: | |
| pass | |
| finally: | |
| sums_file.close() | |
| def FilterUnchangedFiles(self, files): | |
| changed_or_new = [] | |
| for file in files: | |
| try: | |
| handle = open(file, "r") | |
| file_sum = md5er(handle.read()).digest() | |
| if not file in self.sums or self.sums[file] != file_sum: | |
| changed_or_new.append(file) | |
| self.sums[file] = file_sum | |
| finally: | |
| handle.close() | |
| return changed_or_new | |
| def RemoveFile(self, file): | |
| if file in self.sums: | |
| self.sums.pop(file) | |
| class SourceFileProcessor(object): | |
| """ | |
| Utility class that can run through a directory structure, find all relevant | |
| files and invoke a custom check on the files. | |
| """ | |
| def RunOnPath(self, path): | |
| """Runs processor on all files under the given path.""" | |
| all_files = [] | |
| for file in self.GetPathsToSearch(): | |
| all_files += self.FindFilesIn(join(path, file)) | |
| return self.ProcessFiles(all_files) | |
| def RunOnFiles(self, files): | |
| """Runs processor only on affected files.""" | |
| # Helper for getting directory pieces. | |
| dirs = lambda f: dirname(f).split(os.sep) | |
| # Path offsets where to look (to be in sync with RunOnPath). | |
| # Normalize '.' to check for it with str.startswith. | |
| search_paths = [('' if p == '.' else p) for p in self.GetPathsToSearch()] | |
| all_files = [ | |
| f.AbsoluteLocalPath() | |
| for f in files | |
| if (not self.IgnoreFile(f.LocalPath()) and | |
| self.IsRelevant(f.LocalPath()) and | |
| all(not self.IgnoreDir(d) for d in dirs(f.LocalPath())) and | |
| any(map(f.LocalPath().startswith, search_paths))) | |
| ] | |
| return self.ProcessFiles(all_files) | |
| def IgnoreDir(self, name): | |
| return (name.startswith('.') or | |
| name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken', | |
| 'octane', 'sunspider', 'traces-arm64')) | |
| def IgnoreFile(self, name): | |
| return name.startswith('.') | |
| def FindFilesIn(self, path): | |
| result = [] | |
| for (root, dirs, files) in os.walk(path): | |
| for ignored in [x for x in dirs if self.IgnoreDir(x)]: | |
| dirs.remove(ignored) | |
| for file in files: | |
| if not self.IgnoreFile(file) and self.IsRelevant(file): | |
| result.append(join(root, file)) | |
| return result | |
| class CppLintProcessor(SourceFileProcessor): | |
| """ | |
| Lint files to check that they follow the google code style. | |
| """ | |
| def IsRelevant(self, name): | |
| return name.endswith('.cc') or name.endswith('.h') | |
| def IgnoreDir(self, name): | |
| return (super(CppLintProcessor, self).IgnoreDir(name) | |
| or (name == 'third_party')) | |
| IGNORE_LINT = ['export-template.h', 'flag-definitions.h'] | |
| def IgnoreFile(self, name): | |
| return (super(CppLintProcessor, self).IgnoreFile(name) | |
| or (name in CppLintProcessor.IGNORE_LINT)) | |
| def GetPathsToSearch(self): | |
| dirs = ['include', 'samples', 'src'] | |
| test_dirs = ['cctest', 'common', 'fuzzer', 'inspector', 'unittests'] | |
| return dirs + [join('test', dir) for dir in test_dirs] | |
| def GetCpplintScript(self, prio_path): | |
| for path in [prio_path] + os.environ["PATH"].split(os.pathsep): | |
| path = path.strip('"') | |
| cpplint = os.path.join(path, "cpplint.py") | |
| if os.path.isfile(cpplint): | |
| return cpplint | |
| return None | |
| def ProcessFiles(self, files): | |
| good_files_cache = FileContentsCache('.cpplint-cache') | |
| good_files_cache.Load() | |
| files = good_files_cache.FilterUnchangedFiles(files) | |
| if len(files) == 0: | |
| print 'No changes in files detected. Skipping cpplint check.' | |
| return True | |
| filters = ",".join([n for n in LINT_RULES]) | |
| cpplint = self.GetCpplintScript(TOOLS_PATH) | |
| if cpplint is None: | |
| print('Could not find cpplint.py. Make sure ' | |
| 'depot_tools is installed and in the path.') | |
| sys.exit(1) | |
| command = [sys.executable, cpplint, '--filter', filters] | |
| commands = [command + [file] for file in files] | |
| count = multiprocessing.cpu_count() | |
| pool = multiprocessing.Pool(count) | |
| try: | |
| results = pool.map_async(CppLintWorker, commands).get(999999) | |
| except KeyboardInterrupt: | |
| print "\nCaught KeyboardInterrupt, terminating workers." | |
| sys.exit(1) | |
| for i in range(len(files)): | |
| if results[i] > 0: | |
| good_files_cache.RemoveFile(files[i]) | |
| total_errors = sum(results) | |
| print "Total errors found: %d" % total_errors | |
| good_files_cache.Save() | |
| return total_errors == 0 | |
| COPYRIGHT_HEADER_PATTERN = re.compile( | |
| r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.') | |
| class SourceProcessor(SourceFileProcessor): | |
| """ | |
| Check that all files include a copyright notice and no trailing whitespaces. | |
| """ | |
| RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', | |
| '.status', '.gyp', '.gypi'] | |
| def __init__(self): | |
| self.runtime_function_call_pattern = self.CreateRuntimeFunctionCallMatcher() | |
| def CreateRuntimeFunctionCallMatcher(self): | |
| runtime_h_path = join(dirname(TOOLS_PATH), 'src/runtime/runtime.h') | |
| pattern = re.compile(r'\s+F\(([^,]*),.*\)') | |
| runtime_functions = [] | |
| with open(runtime_h_path) as f: | |
| for line in f.readlines(): | |
| m = pattern.match(line) | |
| if m: | |
| runtime_functions.append(m.group(1)) | |
| if len(runtime_functions) < 500: | |
| print ("Runtime functions list is suspiciously short. " | |
| "Consider updating the presubmit script.") | |
| sys.exit(1) | |
| str = '(\%\s+(' + '|'.join(runtime_functions) + '))[\s\(]' | |
| return re.compile(str) | |
| # Overwriting the one in the parent class. | |
| def FindFilesIn(self, path): | |
| if os.path.exists(path+'/.git'): | |
| output = subprocess.Popen('git ls-files --full-name', | |
| stdout=PIPE, cwd=path, shell=True) | |
| result = [] | |
| for file in output.stdout.read().split(): | |
| for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'): | |
| if self.IgnoreDir(dir_part): | |
| break | |
| else: | |
| if (self.IsRelevant(file) and os.path.exists(file) | |
| and not self.IgnoreFile(file)): | |
| result.append(join(path, file)) | |
| if output.wait() == 0: | |
| return result | |
| return super(SourceProcessor, self).FindFilesIn(path) | |
| def IsRelevant(self, name): | |
| for ext in SourceProcessor.RELEVANT_EXTENSIONS: | |
| if name.endswith(ext): | |
| return True | |
| return False | |
| def GetPathsToSearch(self): | |
| return ['.'] | |
| def IgnoreDir(self, name): | |
| return (super(SourceProcessor, self).IgnoreDir(name) or | |
| name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources')) | |
| IGNORE_COPYRIGHTS = ['box2d.js', | |
| 'cpplint.py', | |
| 'check_injected_script_source.py', | |
| 'copy.js', | |
| 'corrections.js', | |
| 'crypto.js', | |
| 'daemon.py', | |
| 'debugger-script.js', | |
| 'earley-boyer.js', | |
| 'fannkuch.js', | |
| 'fasta.js', | |
| 'generate_protocol_externs.py', | |
| 'injected-script.cc', | |
| 'injected-script.h', | |
| 'injected-script-source.js', | |
| 'java-script-call-frame.cc', | |
| 'java-script-call-frame.h', | |
| 'jsmin.py', | |
| 'libraries.cc', | |
| 'libraries-empty.cc', | |
| 'lua_binarytrees.js', | |
| 'meta-123.js', | |
| 'memops.js', | |
| 'poppler.js', | |
| 'primes.js', | |
| 'raytrace.js', | |
| 'regexp-pcre.js', | |
| 'resources-123.js', | |
| 'rjsmin.py', | |
| 'sqlite.js', | |
| 'sqlite-change-heap.js', | |
| 'sqlite-pointer-masking.js', | |
| 'sqlite-safe-heap.js', | |
| 'v8-debugger-script.h', | |
| 'v8-function-call.cc', | |
| 'v8-function-call.h', | |
| 'v8-inspector-impl.cc', | |
| 'v8-inspector-impl.h', | |
| 'v8-runtime-agent-impl.cc', | |
| 'v8-runtime-agent-impl.h', | |
| 'gnuplot-4.6.3-emscripten.js', | |
| 'zlib.js'] | |
| IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js'] | |
| IGNORE_COPYRIGHTS_DIRECTORY = "test/test262/local-tests" | |
| def EndOfDeclaration(self, line): | |
| return line == "}" or line == "};" | |
| def StartOfDeclaration(self, line): | |
| return line.find("//") == 0 or \ | |
| line.find("/*") == 0 or \ | |
| line.find(") {") != -1 | |
| def ProcessContents(self, name, contents): | |
| result = True | |
| base = basename(name) | |
| if not base in SourceProcessor.IGNORE_TABS: | |
| if '\t' in contents: | |
| print "%s contains tabs" % name | |
| result = False | |
| if not base in SourceProcessor.IGNORE_COPYRIGHTS and \ | |
| not SourceProcessor.IGNORE_COPYRIGHTS_DIRECTORY in name: | |
| if not COPYRIGHT_HEADER_PATTERN.search(contents): | |
| print "%s is missing a correct copyright header." % name | |
| result = False | |
| if ' \n' in contents or contents.endswith(' '): | |
| line = 0 | |
| lines = [] | |
| parts = contents.split(' \n') | |
| if not contents.endswith(' '): | |
| parts.pop() | |
| for part in parts: | |
| line += part.count('\n') + 1 | |
| lines.append(str(line)) | |
| linenumbers = ', '.join(lines) | |
| if len(lines) > 1: | |
| print "%s has trailing whitespaces in lines %s." % (name, linenumbers) | |
| else: | |
| print "%s has trailing whitespaces in line %s." % (name, linenumbers) | |
| result = False | |
| if not contents.endswith('\n') or contents.endswith('\n\n'): | |
| print "%s does not end with a single new line." % name | |
| result = False | |
| # Sanitize flags for fuzzer. | |
| if "mjsunit" in name or "debugger" in name: | |
| match = FLAGS_LINE.search(contents) | |
| if match: | |
| print "%s Flags should use '-' (not '_')" % name | |
| result = False | |
| if not "mjsunit/mjsunit.js" in name: | |
| if ASSERT_OPTIMIZED_PATTERN.search(contents) and \ | |
| not FLAGS_ENABLE_OPT.search(contents): | |
| print "%s Flag --opt should be set if " \ | |
| "assertOptimized() is used" % name | |
| result = False | |
| if ASSERT_UNOPTIMIZED_PATTERN.search(contents) and \ | |
| not FLAGS_NO_ALWAYS_OPT.search(contents): | |
| print "%s Flag --no-always-opt should be set if " \ | |
| "assertUnoptimized() is used" % name | |
| result = False | |
| match = self.runtime_function_call_pattern.search(contents) | |
| if match: | |
| print "%s has unexpected spaces in a runtime call '%s'" % (name, match.group(1)) | |
| result = False | |
| return result | |
| def ProcessFiles(self, files): | |
| success = True | |
| violations = 0 | |
| for file in files: | |
| try: | |
| handle = open(file) | |
| contents = handle.read() | |
| if not self.ProcessContents(file, contents): | |
| success = False | |
| violations += 1 | |
| finally: | |
| handle.close() | |
| print "Total violating files: %s" % violations | |
| return success | |
| def _CheckStatusFileForDuplicateKeys(filepath): | |
| comma_space_bracket = re.compile(", *]") | |
| lines = [] | |
| with open(filepath) as f: | |
| for line in f.readlines(): | |
| # Skip all-comment lines. | |
| if line.lstrip().startswith("#"): continue | |
| # Strip away comments at the end of the line. | |
| comment_start = line.find("#") | |
| if comment_start != -1: | |
| line = line[:comment_start] | |
| line = line.strip() | |
| # Strip away trailing commas within the line. | |
| line = comma_space_bracket.sub("]", line) | |
| if len(line) > 0: | |
| lines.append(line) | |
| # Strip away trailing commas at line ends. Ugh. | |
| for i in range(len(lines) - 1): | |
| if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and | |
| lines[i + 1][0] in ("}", "]")): | |
| lines[i] = lines[i][:-1] | |
| contents = "\n".join(lines) | |
| # JSON wants double-quotes. | |
| contents = contents.replace("'", '"') | |
| # Fill in keywords (like PASS, SKIP). | |
| for key in statusfile.KEYWORDS: | |
| contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents) | |
| status = {"success": True} | |
| def check_pairs(pairs): | |
| keys = {} | |
| for key, value in pairs: | |
| if key in keys: | |
| print("%s: Error: duplicate key %s" % (filepath, key)) | |
| status["success"] = False | |
| keys[key] = True | |
| json.loads(contents, object_pairs_hook=check_pairs) | |
| return status["success"] | |
| class StatusFilesProcessor(SourceFileProcessor): | |
| """Checks status files for incorrect syntax and duplicate keys.""" | |
| def IsRelevant(self, name): | |
| # Several changes to files under the test directories could impact status | |
| # files. | |
| return True | |
| def GetPathsToSearch(self): | |
| return ['test', 'tools/testrunner'] | |
| def ProcessFiles(self, files): | |
| success = True | |
| for status_file_path in sorted(self._GetStatusFiles(files)): | |
| success &= statusfile.PresubmitCheck(status_file_path) | |
| success &= _CheckStatusFileForDuplicateKeys(status_file_path) | |
| return success | |
| def _GetStatusFiles(self, files): | |
| test_path = join(dirname(TOOLS_PATH), 'test') | |
| testrunner_path = join(TOOLS_PATH, 'testrunner') | |
| status_files = set() | |
| for file_path in files: | |
| if file_path.startswith(testrunner_path): | |
| for suitepath in os.listdir(test_path): | |
| suitename = os.path.basename(suitepath) | |
| status_file = os.path.join( | |
| test_path, suitename, suitename + ".status") | |
| if os.path.exists(status_file): | |
| status_files.add(status_file) | |
| return status_files | |
| for file_path in files: | |
| if file_path.startswith(test_path): | |
| # Strip off absolute path prefix pointing to test suites. | |
| pieces = file_path[len(test_path):].lstrip(os.sep).split(os.sep) | |
| if pieces: | |
| # Infer affected status file name. Only care for existing status | |
| # files. Some directories under "test" don't have any. | |
| if not os.path.isdir(join(test_path, pieces[0])): | |
| continue | |
| status_file = join(test_path, pieces[0], pieces[0] + ".status") | |
| if not os.path.exists(status_file): | |
| continue | |
| status_files.add(status_file) | |
| return status_files | |
| def CheckDeps(workspace): | |
| checkdeps_py = join(workspace, 'buildtools', 'checkdeps', 'checkdeps.py') | |
| return subprocess.call([sys.executable, checkdeps_py, workspace]) == 0 | |
| def PyTests(workspace): | |
| result = True | |
| for script in [ | |
| join(workspace, 'tools', 'release', 'test_scripts.py'), | |
| join(workspace, 'tools', 'unittests', 'run_tests_test.py'), | |
| ]: | |
| print 'Running ' + script | |
| result &= subprocess.call( | |
| [sys.executable, script], stdout=subprocess.PIPE) == 0 | |
| return result | |
| def GetOptions(): | |
| result = optparse.OptionParser() | |
| result.add_option('--no-lint', help="Do not run cpplint", default=False, | |
| action="store_true") | |
| return result | |
| def Main(): | |
| workspace = abspath(join(dirname(sys.argv[0]), '..')) | |
| parser = GetOptions() | |
| (options, args) = parser.parse_args() | |
| success = True | |
| print "Running checkdeps..." | |
| success &= CheckDeps(workspace) | |
| if not options.no_lint: | |
| print "Running C++ lint check..." | |
| success &= CppLintProcessor().RunOnPath(workspace) | |
| print "Running copyright header, trailing whitespaces and " \ | |
| "two empty lines between declarations check..." | |
| success &= SourceProcessor().RunOnPath(workspace) | |
| print "Running status-files check..." | |
| success &= StatusFilesProcessor().RunOnPath(workspace) | |
| print "Running python tests..." | |
| success &= PyTests(workspace) | |
| if success: | |
| return 0 | |
| else: | |
| return 1 | |
| if __name__ == '__main__': | |
| sys.exit(Main()) |