Parallelize pylint PRESUBMIT checks.
R=maruel@chromium.org
BUG=479837,499650
Review URL: https://codereview.chromium.org/1181103002
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295664 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/presubmit_canned_checks.py b/presubmit_canned_checks.py
index 9179fb6..edd6332 100644
--- a/presubmit_canned_checks.py
+++ b/presubmit_canned_checks.py
@@ -771,20 +771,28 @@
env['PYTHONPATH'] = input_api.os_path.pathsep.join(
extra_paths_list + sys.path).encode('utf8')
- def GetPylintCmd(files):
+ def GetPylintCmd(files, extra, parallel):
# Windows needs help running python files so we explicitly specify
# the interpreter to use. It also has limitations on the size of
# the command-line, so we pass arguments via a pipe.
+ cmd = [input_api.python_executable,
+ input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
+ '--args-on-stdin']
if len(files) == 1:
description = files[0]
else:
description = '%s files' % len(files)
+ if extra:
+ cmd.extend(extra)
+ description += ' using %s' % (extra,)
+ if parallel:
+ cmd.append('--jobs=%s' % input_api.cpu_count)
+ description += ' on %d cores' % input_api.cpu_count
+
return input_api.Command(
name='Pylint (%s)' % description,
- cmd=[input_api.python_executable,
- input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
- '--args-on-stdin'],
+ cmd=cmd,
kwargs={'env': env, 'stdin': '\n'.join(files + extra_args)},
message=error_type)
@@ -797,9 +805,14 @@
# a quick local edit to diagnose pylint issues more
# easily.
if True:
- return [GetPylintCmd(files)]
+ # pylint's cycle detection doesn't work in parallel, so spawn a second,
+ # single-threaded job for just that check.
+ return [
+ GetPylintCmd(files, ["--disable=cyclic-import"], True),
+ GetPylintCmd(files, ["--disable=all", "--enable=cyclic-import"], False)
+ ]
else:
- return map(lambda x: GetPylintCmd([x]), files)
+ return map(lambda x: GetPylintCmd([x], extra_args, 1), files)
def RunPylint(input_api, *args, **kwargs):
diff --git a/presubmit_support.py b/presubmit_support.py
index 0abd421..4baee77 100755
--- a/presubmit_support.py
+++ b/presubmit_support.py
@@ -311,6 +311,8 @@
# InputApi.platform is the platform you're currently running on.
self.platform = sys.platform
+ self.cpu_count = multiprocessing.cpu_count()
+
# The local path of the currently-being-processed presubmit script.
self._current_presubmit_path = os.path.dirname(presubmit_path)
diff --git a/tests/presubmit_unittest.py b/tests/presubmit_unittest.py
index 0d2b4d6..f37bf6c 100755
--- a/tests/presubmit_unittest.py
+++ b/tests/presubmit_unittest.py
@@ -1183,17 +1183,15 @@
self.mox.ReplayAll()
members = [
'AbsoluteLocalPaths', 'AffectedFiles', 'AffectedSourceFiles',
- 'AffectedTextFiles',
- 'DEFAULT_BLACK_LIST', 'DEFAULT_WHITE_LIST',
- 'DepotToLocalPath', 'FilterSourceFile', 'LocalPaths',
- 'LocalToDepotPath', 'Command', 'RunTests',
- 'PresubmitLocalPath', 'ReadFile', 'RightHandSideLines', 'ServerPaths',
- 'basename', 'cPickle', 'cpplint', 'cStringIO', 'canned_checks', 'change',
- 'environ', 'glob', 'host_url', 'is_committing', 'json', 'logging',
- 'marshal', 'os_listdir', 'os_walk', 'os_path', 'os_stat', 'owners_db',
- 'pickle', 'platform', 'python_executable', 're', 'rietveld', 'subprocess',
- 'tbr', 'tempfile', 'time', 'traceback', 'unittest', 'urllib2', 'version',
- 'verbose',
+ 'AffectedTextFiles', 'DEFAULT_BLACK_LIST', 'DEFAULT_WHITE_LIST',
+ 'DepotToLocalPath', 'FilterSourceFile', 'LocalPaths', 'LocalToDepotPath',
+ 'Command', 'RunTests', 'PresubmitLocalPath', 'ReadFile',
+ 'RightHandSideLines', 'ServerPaths', 'basename', 'cPickle', 'cpplint',
+ 'cStringIO', 'canned_checks', 'change', 'cpu_count', 'environ', 'glob',
+ 'host_url', 'is_committing', 'json', 'logging', 'marshal', 'os_listdir',
+ 'os_walk', 'os_path', 'os_stat', 'owners_db', 'pickle', 'platform',
+ 'python_executable', 're', 'rietveld', 'subprocess', 'tbr', 'tempfile',
+ 'time', 'traceback', 'unittest', 'urllib2', 'version', 'verbose',
]
# If this test fails, you should add the relevant test.
self.compareMembers(
@@ -1852,6 +1850,7 @@
input_api.tbr = False
input_api.python_executable = 'pyyyyython'
input_api.platform = sys.platform
+ input_api.cpu_count = 2
input_api.time = time
input_api.canned_checks = presubmit_canned_checks
input_api.Command = presubmit.CommandData
@@ -2525,7 +2524,12 @@
pylintrc = os.path.join(_ROOT, 'pylintrc')
CommHelper(input_api,
- ['pyyyyython', pylint, '--args-on-stdin'],
+ ['pyyyyython', pylint, '--args-on-stdin', '--disable=cyclic-import',
+ '--jobs=2'],
+ env=mox.IgnoreArg(), stdin='file1.py\n--rcfile=%s' % pylintrc)
+ CommHelper(input_api,
+ ['pyyyyython', pylint, '--args-on-stdin', '--disable=all',
+ '--enable=cyclic-import'],
env=mox.IgnoreArg(), stdin='file1.py\n--rcfile=%s' % pylintrc)
self.mox.ReplayAll()
