| # Copyright (c) 2003-2014 LOGILAB S.A. (Paris, FRANCE). |
| # http://www.logilab.fr/ -- mailto:contact@logilab.fr |
| # |
| # This program is free software; you can redistribute it and/or modify it under |
| # the terms of the GNU General Public License as published by the Free Software |
| # Foundation; either version 2 of the License, or (at your option) any later |
| # version. |
| # |
| # This program is distributed in the hope that it will be useful, but WITHOUT |
| # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
| # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details |
| # |
| # You should have received a copy of the GNU General Public License along with |
| # this program; if not, write to the Free Software Foundation, Inc., |
| # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| """ %prog [options] module_or_package |
| |
| Check that a module satisfies a coding standard (and more !). |
| |
| %prog --help |
| |
| Display this help message and exit. |
| |
| %prog --help-msg <msg-id>[,<msg-id>] |
| |
| Display help messages about given message identifiers and exit. |
| """ |
| from __future__ import print_function |
| |
| import collections |
| import contextlib |
| import itertools |
| import operator |
| import os |
| try: |
| import multiprocessing |
| except ImportError: |
| multiprocessing = None |
| import sys |
| import tokenize |
| import warnings |
| |
| import astroid |
| from astroid.__pkginfo__ import version as astroid_version |
| from astroid import modutils |
| from logilab.common import configuration |
| from logilab.common import optik_ext |
| from logilab.common import interface |
| from logilab.common import textutils |
| from logilab.common import ureports |
| from logilab.common.__pkginfo__ import version as common_version |
| import six |
| |
| from pylint import checkers |
| from pylint import interfaces |
| from pylint import reporters |
| from pylint import utils |
| from pylint import config |
| from pylint.__pkginfo__ import version |
| |
| |
| MANAGER = astroid.MANAGER |
| |
| def _get_new_args(message): |
| location = ( |
| message.abspath, |
| message.path, |
| message.module, |
| message.obj, |
| message.line, |
| message.column, |
| ) |
| return ( |
| message.msg_id, |
| message.symbol, |
| location, |
| message.msg, |
| message.confidence, |
| ) |
| |
| def _get_python_path(filepath): |
| dirname = os.path.realpath(os.path.expanduser(filepath)) |
| if not os.path.isdir(dirname): |
| dirname = os.path.dirname(dirname) |
| while True: |
| if not os.path.exists(os.path.join(dirname, "__init__.py")): |
| return dirname |
| old_dirname = dirname |
| dirname = os.path.dirname(dirname) |
| if old_dirname == dirname: |
| return os.getcwd() |
| |
| |
| def _merge_stats(stats): |
| merged = {} |
| for stat in stats: |
| for key, item in six.iteritems(stat): |
| if key not in merged: |
| merged[key] = item |
| else: |
| if isinstance(item, dict): |
| merged[key].update(item) |
| else: |
| merged[key] = merged[key] + item |
| return merged |
| |
| |
| # Python Linter class ######################################################### |
| |
| MSGS = { |
| 'F0001': ('%s', |
| 'fatal', |
| 'Used when an error occurred preventing the analysis of a \ |
| module (unable to find it for instance).'), |
| 'F0002': ('%s: %s', |
| 'astroid-error', |
| 'Used when an unexpected error occurred while building the ' |
| 'Astroid representation. This is usually accompanied by a ' |
| 'traceback. Please report such errors !'), |
| 'F0003': ('ignored builtin module %s', |
| 'ignored-builtin-module', |
| 'Used to indicate that the user asked to analyze a builtin ' |
| 'module which has been skipped.'), |
| 'F0010': ('error while code parsing: %s', |
| 'parse-error', |
| 'Used when an exception occured while building the Astroid ' |
| 'representation which could be handled by astroid.'), |
| |
| 'I0001': ('Unable to run raw checkers on built-in module %s', |
| 'raw-checker-failed', |
| 'Used to inform that a built-in module has not been checked ' |
| 'using the raw checkers.'), |
| |
| 'I0010': ('Unable to consider inline option %r', |
| 'bad-inline-option', |
| 'Used when an inline option is either badly formatted or can\'t ' |
| 'be used inside modules.'), |
| |
| 'I0011': ('Locally disabling %s (%s)', |
| 'locally-disabled', |
| 'Used when an inline option disables a message or a messages ' |
| 'category.'), |
| 'I0012': ('Locally enabling %s (%s)', |
| 'locally-enabled', |
| 'Used when an inline option enables a message or a messages ' |
| 'category.'), |
| 'I0013': ('Ignoring entire file', |
| 'file-ignored', |
| 'Used to inform that the file will not be checked'), |
| 'I0020': ('Suppressed %s (from line %d)', |
| 'suppressed-message', |
| 'A message was triggered on a line, but suppressed explicitly ' |
| 'by a disable= comment in the file. This message is not ' |
| 'generated for messages that are ignored due to configuration ' |
| 'settings.'), |
| 'I0021': ('Useless suppression of %s', |
| 'useless-suppression', |
| 'Reported when a message is explicitly disabled for a line or ' |
| 'a block of code, but never triggered.'), |
| 'I0022': ('Pragma "%s" is deprecated, use "%s" instead', |
| 'deprecated-pragma', |
| 'Some inline pylint options have been renamed or reworked, ' |
| 'only the most recent form should be used. ' |
| 'NOTE:skip-all is only available with pylint >= 0.26', |
| {'old_names': [('I0014', 'deprecated-disable-all')]}), |
| |
| 'E0001': ('%s', |
| 'syntax-error', |
| 'Used when a syntax error is raised for a module.'), |
| |
| 'E0011': ('Unrecognized file option %r', |
| 'unrecognized-inline-option', |
| 'Used when an unknown inline option is encountered.'), |
| 'E0012': ('Bad option value %r', |
| 'bad-option-value', |
| 'Used when a bad value for an inline option is encountered.'), |
| } |
| |
| |
| def _deprecated_option(shortname, opt_type): |
| def _warn_deprecated(option, optname, *args): # pylint: disable=unused-argument |
| sys.stderr.write('Warning: option %s is deprecated and ignored.\n' % (optname,)) |
| return {'short': shortname, 'help': 'DEPRECATED', 'hide': True, |
| 'type': opt_type, 'action': 'callback', 'callback': _warn_deprecated} |
| |
| |
| if multiprocessing is not None: |
| class ChildLinter(multiprocessing.Process): # pylint: disable=no-member |
| def run(self): |
| tasks_queue, results_queue, self._config = self._args # pylint: disable=no-member |
| |
| self._config["jobs"] = 1 # Child does not parallelize any further. |
| |
| # Run linter for received files/modules. |
| for file_or_module in iter(tasks_queue.get, 'STOP'): |
| result = self._run_linter(file_or_module[0]) |
| try: |
| results_queue.put(result) |
| except Exception as ex: |
| print("internal error with sending report for module %s" % file_or_module, file=sys.stderr) |
| print(ex, file=sys.stderr) |
| results_queue.put({}) |
| |
| def _run_linter(self, file_or_module): |
| linter = PyLinter() |
| |
| # Register standard checkers. |
| linter.load_default_plugins() |
| # Load command line plugins. |
| # TODO linter.load_plugin_modules(self._plugins) |
| |
| linter.load_configuration(**self._config) |
| linter.set_reporter(reporters.CollectingReporter()) |
| |
| # Run the checks. |
| linter.check(file_or_module) |
| |
| msgs = [_get_new_args(m) for m in linter.reporter.messages] |
| return (file_or_module, linter.file_state.base_name, linter.current_name, |
| msgs, linter.stats, linter.msg_status) |
| |
| |
| class PyLinter(configuration.OptionsManagerMixIn, |
| utils.MessagesHandlerMixIn, |
| utils.ReportsHandlerMixIn, |
| checkers.BaseTokenChecker): |
| """lint Python modules using external checkers. |
| |
| This is the main checker controlling the other ones and the reports |
| generation. It is itself both a raw checker and an astroid checker in order |
| to: |
| * handle message activation / deactivation at the module level |
| * handle some basic but necessary stats'data (number of classes, methods...) |
| |
| IDE plugins developpers: you may have to call |
| `astroid.builder.MANAGER.astroid_cache.clear()` accross run if you want |
| to ensure the latest code version is actually checked. |
| """ |
| |
| __implements__ = (interfaces.ITokenChecker, ) |
| |
| name = 'master' |
| priority = 0 |
| level = 0 |
| msgs = MSGS |
| |
| @staticmethod |
| def make_options(): |
| return (('ignore', |
| {'type' : 'csv', 'metavar' : '<file>[,<file>...]', |
| 'dest' : 'black_list', 'default' : ('CVS',), |
| 'help' : 'Add files or directories to the blacklist. ' |
| 'They should be base names, not paths.'}), |
| ('persistent', |
| {'default': True, 'type' : 'yn', 'metavar' : '<y_or_n>', |
| 'level': 1, |
| 'help' : 'Pickle collected data for later comparisons.'}), |
| |
| ('load-plugins', |
| {'type' : 'csv', 'metavar' : '<modules>', 'default' : (), |
| 'level': 1, |
| 'help' : 'List of plugins (as comma separated values of ' |
| 'python modules names) to load, usually to register ' |
| 'additional checkers.'}), |
| |
| ('output-format', |
| {'default': 'text', 'type': 'string', 'metavar' : '<format>', |
| 'short': 'f', |
| 'group': 'Reports', |
| 'help' : 'Set the output format. Available formats are text,' |
| ' parseable, colorized, msvs (visual studio) and html. You ' |
| 'can also give a reporter class, eg mypackage.mymodule.' |
| 'MyReporterClass.'}), |
| |
| ('files-output', |
| {'default': 0, 'type' : 'yn', 'metavar' : '<y_or_n>', |
| 'group': 'Reports', 'level': 1, |
| 'help' : 'Put messages in a separate file for each module / ' |
| 'package specified on the command line instead of printing ' |
| 'them on stdout. Reports (if any) will be written in a file ' |
| 'name "pylint_global.[txt|html]".'}), |
| |
| ('reports', |
| {'default': 1, 'type' : 'yn', 'metavar' : '<y_or_n>', |
| 'short': 'r', |
| 'group': 'Reports', |
| 'help' : 'Tells whether to display a full report or only the ' |
| 'messages'}), |
| |
| ('evaluation', |
| {'type' : 'string', 'metavar' : '<python_expression>', |
| 'group': 'Reports', 'level': 1, |
| 'default': '10.0 - ((float(5 * error + warning + refactor + ' |
| 'convention) / statement) * 10)', |
| 'help' : 'Python expression which should return a note less ' |
| 'than 10 (10 is the highest note). You have access ' |
| 'to the variables errors warning, statement which ' |
| 'respectively contain the number of errors / ' |
| 'warnings messages and the total number of ' |
| 'statements analyzed. This is used by the global ' |
| 'evaluation report (RP0004).'}), |
| |
| ('comment', |
| {'default': 0, 'type' : 'yn', 'metavar' : '<y_or_n>', |
| 'group': 'Reports', 'level': 1, |
| 'help' : 'Add a comment according to your evaluation note. ' |
| 'This is used by the global evaluation report (RP0004).'}), |
| |
| ('confidence', |
| {'type' : 'multiple_choice', 'metavar': '<levels>', |
| 'default': '', |
| 'choices': [c.name for c in interfaces.CONFIDENCE_LEVELS], |
| 'group': 'Messages control', |
| 'help' : 'Only show warnings with the listed confidence levels.' |
| ' Leave empty to show all. Valid levels: %s' % ( |
| ', '.join(c.name for c in interfaces.CONFIDENCE_LEVELS),)}), |
| |
| ('enable', |
| {'type' : 'csv', 'metavar': '<msg ids>', |
| 'short': 'e', |
| 'group': 'Messages control', |
| 'help' : 'Enable the message, report, category or checker with the ' |
| 'given id(s). You can either give multiple identifier ' |
| 'separated by comma (,) or put this option multiple time. ' |
| 'See also the "--disable" option for examples. '}), |
| |
| ('disable', |
| {'type' : 'csv', 'metavar': '<msg ids>', |
| 'short': 'd', |
| 'group': 'Messages control', |
| 'help' : 'Disable the message, report, category or checker ' |
| 'with the given id(s). You can either give multiple identifiers' |
| ' separated by comma (,) or put this option multiple times ' |
| '(only on the command line, not in the configuration file ' |
| 'where it should appear only once).' |
| 'You can also use "--disable=all" to disable everything first ' |
| 'and then reenable specific checks. For example, if you want ' |
| 'to run only the similarities checker, you can use ' |
| '"--disable=all --enable=similarities". ' |
| 'If you want to run only the classes checker, but have no ' |
| 'Warning level messages displayed, use' |
| '"--disable=all --enable=classes --disable=W"'}), |
| |
| ('msg-template', |
| {'type' : 'string', 'metavar': '<template>', |
| 'group': 'Reports', |
| 'help' : ('Template used to display messages. ' |
| 'This is a python new-style format string ' |
| 'used to format the message information. ' |
| 'See doc for all details') |
| }), |
| |
| ('include-ids', _deprecated_option('i', 'yn')), |
| ('symbols', _deprecated_option('s', 'yn')), |
| |
| ('jobs', |
| {'type' : 'int', 'metavar': '<n-processes>', |
| 'short': 'j', |
| 'default': 1, |
| 'help' : '''Use multiple processes to speed up Pylint.''', |
| }), |
| |
| ('unsafe-load-any-extension', |
| {'type': 'yn', 'metavar': '<yn>', 'default': False, 'hide': True, |
| 'help': ('Allow loading of arbitrary C extensions. Extensions' |
| ' are imported into the active Python interpreter and' |
| ' may run arbitrary code.')}), |
| |
| ('extension-pkg-whitelist', |
| {'type': 'csv', 'metavar': '<pkg[,pkg]>', 'default': [], |
| 'help': ('A comma-separated list of package or module names' |
| ' from where C extensions may be loaded. Extensions are' |
| ' loading into the active Python interpreter and may run' |
| ' arbitrary code')} |
| ), |
| ) |
| |
| option_groups = ( |
| ('Messages control', 'Options controling analysis messages'), |
| ('Reports', 'Options related to output formating and reporting'), |
| ) |
| |
| def __init__(self, options=(), reporter=None, option_groups=(), |
| pylintrc=None): |
| # some stuff has to be done before ancestors initialization... |
| # |
| # messages store / checkers / reporter / astroid manager |
| self.msgs_store = utils.MessagesStore() |
| self.reporter = None |
| self._reporter_name = None |
| self._reporters = {} |
| self._checkers = collections.defaultdict(list) |
| self._pragma_lineno = {} |
| self._ignore_file = False |
| # visit variables |
| self.file_state = utils.FileState() |
| self.current_name = None |
| self.current_file = None |
| self.stats = None |
| # init options |
| self._external_opts = options |
| self.options = options + PyLinter.make_options() |
| self.option_groups = option_groups + PyLinter.option_groups |
| self._options_methods = { |
| 'enable': self.enable, |
| 'disable': self.disable} |
| self._bw_options_methods = {'disable-msg': self.disable, |
| 'enable-msg': self.enable} |
| full_version = '%%prog %s, \nastroid %s, common %s\nPython %s' % ( |
| version, astroid_version, common_version, sys.version) |
| configuration.OptionsManagerMixIn.__init__( |
| self, usage=__doc__, |
| version=full_version, |
| config_file=pylintrc or config.PYLINTRC) |
| utils.MessagesHandlerMixIn.__init__(self) |
| utils.ReportsHandlerMixIn.__init__(self) |
| checkers.BaseTokenChecker.__init__(self) |
| # provided reports |
| self.reports = (('RP0001', 'Messages by category', |
| report_total_messages_stats), |
| ('RP0002', '% errors / warnings by module', |
| report_messages_by_module_stats), |
| ('RP0003', 'Messages', |
| report_messages_stats), |
| ('RP0004', 'Global evaluation', |
| self.report_evaluation), |
| ) |
| self.register_checker(self) |
| self._dynamic_plugins = set() |
| self.load_provider_defaults() |
| if reporter: |
| self.set_reporter(reporter) |
| |
| def load_default_plugins(self): |
| checkers.initialize(self) |
| reporters.initialize(self) |
| # Make sure to load the default reporter, because |
| # the option has been set before the plugins had been loaded. |
| if not self.reporter: |
| self._load_reporter() |
| |
| def load_plugin_modules(self, modnames): |
| """take a list of module names which are pylint plugins and load |
| and register them |
| """ |
| for modname in modnames: |
| if modname in self._dynamic_plugins: |
| continue |
| self._dynamic_plugins.add(modname) |
| module = modutils.load_module_from_name(modname) |
| module.register(self) |
| |
| def _load_reporter(self): |
| name = self._reporter_name.lower() |
| if name in self._reporters: |
| self.set_reporter(self._reporters[name]()) |
| else: |
| qname = self._reporter_name |
| module = modutils.load_module_from_name( |
| modutils.get_module_part(qname)) |
| class_name = qname.split('.')[-1] |
| reporter_class = getattr(module, class_name) |
| self.set_reporter(reporter_class()) |
| |
| def set_reporter(self, reporter): |
| """set the reporter used to display messages and reports""" |
| self.reporter = reporter |
| reporter.linter = self |
| |
| def set_option(self, optname, value, action=None, optdict=None): |
| """overridden from configuration.OptionsProviderMixin to handle some |
| special options |
| """ |
| if optname in self._options_methods or \ |
| optname in self._bw_options_methods: |
| if value: |
| try: |
| meth = self._options_methods[optname] |
| except KeyError: |
| meth = self._bw_options_methods[optname] |
| warnings.warn('%s is deprecated, replace it by %s' % ( |
| optname, optname.split('-')[0]), |
| DeprecationWarning) |
| value = optik_ext.check_csv(None, optname, value) |
| if isinstance(value, (list, tuple)): |
| for _id in value: |
| meth(_id, ignore_unknown=True) |
| else: |
| meth(value) |
| return # no need to call set_option, disable/enable methods do it |
| elif optname == 'output-format': |
| self._reporter_name = value |
| # If the reporters are already available, load |
| # the reporter class. |
| if self._reporters: |
| self._load_reporter() |
| |
| try: |
| checkers.BaseTokenChecker.set_option(self, optname, |
| value, action, optdict) |
| except configuration.UnsupportedAction: |
| print('option %s can\'t be read from config file' % \ |
| optname, file=sys.stderr) |
| |
| def register_reporter(self, reporter_class): |
| self._reporters[reporter_class.name] = reporter_class |
| |
| def report_order(self): |
| reports = sorted(self._reports, key=lambda x: getattr(x, 'name', '')) |
| try: |
| # Remove the current reporter and add it |
| # at the end of the list. |
| reports.pop(reports.index(self)) |
| except ValueError: |
| pass |
| else: |
| reports.append(self) |
| return reports |
| |
| # checkers manipulation methods ############################################ |
| |
| def register_checker(self, checker): |
| """register a new checker |
| |
| checker is an object implementing IRawChecker or / and IAstroidChecker |
| """ |
| assert checker.priority <= 0, 'checker priority can\'t be >= 0' |
| self._checkers[checker.name].append(checker) |
| for r_id, r_title, r_cb in checker.reports: |
| self.register_report(r_id, r_title, r_cb, checker) |
| self.register_options_provider(checker) |
| if hasattr(checker, 'msgs'): |
| self.msgs_store.register_messages(checker) |
| checker.load_defaults() |
| |
| # Register the checker, but disable all of its messages. |
| # TODO(cpopa): we should have a better API for this. |
| if not getattr(checker, 'enabled', True): |
| self.disable(checker.name) |
| |
| def disable_noerror_messages(self): |
| for msgcat, msgids in six.iteritems(self.msgs_store._msgs_by_category): |
| if msgcat == 'E': |
| for msgid in msgids: |
| self.enable(msgid) |
| else: |
| for msgid in msgids: |
| self.disable(msgid) |
| |
| def disable_reporters(self): |
| """disable all reporters""" |
| for reporters in six.itervalues(self._reports): |
| for report_id, _, _ in reporters: |
| self.disable_report(report_id) |
| |
| def error_mode(self): |
| """error mode: enable only errors; no reports, no persistent""" |
| self.disable_noerror_messages() |
| self.disable('miscellaneous') |
| self.set_option('reports', False) |
| self.set_option('persistent', False) |
| |
| # block level option handling ############################################# |
| # |
| # see func_block_disable_msg.py test case for expected behaviour |
| |
| def process_tokens(self, tokens): |
| """process tokens from the current module to search for module/block |
| level options |
| """ |
| control_pragmas = {'disable', 'enable'} |
| for (tok_type, content, start, _, _) in tokens: |
| if tok_type != tokenize.COMMENT: |
| continue |
| match = utils.OPTION_RGX.search(content) |
| if match is None: |
| continue |
| if match.group(1).strip() == "disable-all" or \ |
| match.group(1).strip() == 'skip-file': |
| if match.group(1).strip() == "disable-all": |
| self.add_message('deprecated-pragma', line=start[0], |
| args=('disable-all', 'skip-file')) |
| self.add_message('file-ignored', line=start[0]) |
| self._ignore_file = True |
| return |
| try: |
| opt, value = match.group(1).split('=', 1) |
| except ValueError: |
| self.add_message('bad-inline-option', args=match.group(1).strip(), |
| line=start[0]) |
| continue |
| opt = opt.strip() |
| if opt in self._options_methods or opt in self._bw_options_methods: |
| try: |
| meth = self._options_methods[opt] |
| except KeyError: |
| meth = self._bw_options_methods[opt] |
| # found a "(dis|en)able-msg" pragma deprecated suppresssion |
| self.add_message('deprecated-pragma', line=start[0], args=(opt, opt.replace('-msg', ''))) |
| for msgid in textutils.splitstrip(value): |
| # Add the line where a control pragma was encountered. |
| if opt in control_pragmas: |
| self._pragma_lineno[msgid] = start[0] |
| |
| try: |
| if (opt, msgid) == ('disable', 'all'): |
| self.add_message('deprecated-pragma', line=start[0], args=('disable=all', 'skip-file')) |
| self.add_message('file-ignored', line=start[0]) |
| self._ignore_file = True |
| return |
| meth(msgid, 'module', start[0]) |
| except utils.UnknownMessage: |
| self.add_message('bad-option-value', args=msgid, line=start[0]) |
| else: |
| self.add_message('unrecognized-inline-option', args=opt, line=start[0]) |
| |
| |
| # code checking methods ################################################### |
| |
| def get_checkers(self): |
| """return all available checkers as a list""" |
| return [self] + [c for checkers in six.itervalues(self._checkers) |
| for c in checkers if c is not self] |
| |
| def prepare_checkers(self): |
| """return checkers needed for activated messages and reports""" |
| if not self.config.reports: |
| self.disable_reporters() |
| # get needed checkers |
| neededcheckers = [self] |
| for checker in self.get_checkers()[1:]: |
| # fatal errors should not trigger enable / disabling a checker |
| messages = set(msg for msg in checker.msgs |
| if msg[0] != 'F' and self.is_message_enabled(msg)) |
| if (messages or |
| any(self.report_is_enabled(r[0]) for r in checker.reports)): |
| neededcheckers.append(checker) |
| # Sort checkers by priority |
| neededcheckers = sorted(neededcheckers, |
| key=operator.attrgetter('priority'), |
| reverse=True) |
| return neededcheckers |
| |
| def should_analyze_file(self, modname, path): # pylint: disable=unused-argument, no-self-use |
| """Returns whether or not a module should be checked. |
| |
| This implementation returns True for all python source file, indicating |
| that all files should be linted. |
| |
| Subclasses may override this method to indicate that modules satisfying |
| certain conditions should not be linted. |
| |
| :param str modname: The name of the module to be checked. |
| :param str path: The full path to the source code of the module. |
| :returns: True if the module should be checked. |
| :rtype: bool |
| """ |
| return path.endswith('.py') |
| |
| def check(self, files_or_modules): |
| """main checking entry: check a list of files or modules from their |
| name. |
| """ |
| # initialize msgs_state now that all messages have been registered into |
| # the store |
| for msg in self.msgs_store.messages: |
| if not msg.may_be_emitted(): |
| self._msgs_state[msg.msgid] = False |
| |
| if not isinstance(files_or_modules, (list, tuple)): |
| files_or_modules = (files_or_modules,) |
| |
| if self.config.jobs == 1: |
| with fix_import_path(files_or_modules): |
| self._do_check(files_or_modules) |
| else: |
| # Hack that permits running pylint, on Windows, with -m switch |
| # and with --jobs, as in 'python -2 -m pylint .. --jobs'. |
| # For more details why this is needed, |
| # see Python issue http://bugs.python.org/issue10845. |
| |
| mock_main = __name__ != '__main__' # -m switch |
| if mock_main: |
| sys.modules['__main__'] = sys.modules[__name__] |
| try: |
| self._parallel_check(files_or_modules) |
| finally: |
| if mock_main: |
| sys.modules.pop('__main__') |
| |
| def _parallel_task(self, files_or_modules): |
| # Prepare configuration for child linters. |
| filter_options = {'symbols', 'include-ids', 'long-help'} |
| filter_options.update([opt_name for opt_name, _ in self._external_opts]) |
| config = {} |
| for opt_providers in six.itervalues(self._all_options): |
| for optname, optdict, val in opt_providers.options_and_values(): |
| if optname not in filter_options: |
| config[optname] = configuration.format_option_value(optdict, val) |
| |
| childs = [] |
| manager = multiprocessing.Manager() # pylint: disable=no-member |
| tasks_queue = manager.Queue() # pylint: disable=no-member |
| results_queue = manager.Queue() # pylint: disable=no-member |
| |
| for _ in range(self.config.jobs): |
| cl = ChildLinter(args=(tasks_queue, results_queue, config)) |
| cl.start() # pylint: disable=no-member |
| childs.append(cl) |
| |
| # send files to child linters |
| for files_or_module in files_or_modules: |
| tasks_queue.put([files_or_module]) |
| |
| # collect results from child linters |
| failed = False |
| for _ in files_or_modules: |
| try: |
| result = results_queue.get() |
| except Exception as ex: |
| print("internal error while receiving results from child linter", |
| file=sys.stderr) |
| print(ex, file=sys.stderr) |
| failed = True |
| break |
| yield result |
| |
| # Stop child linters and wait for their completion. |
| for _ in range(self.config.jobs): |
| tasks_queue.put('STOP') |
| for cl in childs: |
| cl.join() |
| |
| if failed: |
| print("Error occured, stopping the linter.", file=sys.stderr) |
| sys.exit(32) |
| |
| def _parallel_check(self, files_or_modules): |
| # Reset stats. |
| self.open() |
| |
| all_stats = [] |
| for result in self._parallel_task(files_or_modules): |
| ( |
| file_or_module, |
| self.file_state.base_name, |
| module, |
| messages, |
| stats, |
| msg_status |
| ) = result |
| |
| if file_or_module == files_or_modules[-1]: |
| last_module = module |
| |
| for msg in messages: |
| msg = utils.Message(*msg) |
| self.set_current_module(module) |
| self.reporter.handle_message(msg) |
| |
| all_stats.append(stats) |
| self.msg_status |= msg_status |
| |
| self.stats = _merge_stats(itertools.chain(all_stats, [self.stats])) |
| self.current_name = last_module |
| |
| # Insert stats data to local checkers. |
| for checker in self.get_checkers(): |
| if checker is not self: |
| checker.stats = self.stats |
| |
| def _do_check(self, files_or_modules): |
| walker = utils.PyLintASTWalker(self) |
| checkers = self.prepare_checkers() |
| tokencheckers = [c for c in checkers |
| if interface.implements(c, interfaces.ITokenChecker) |
| and c is not self] |
| rawcheckers = [c for c in checkers |
| if interface.implements(c, interfaces.IRawChecker)] |
| # notify global begin |
| for checker in checkers: |
| checker.open() |
| if interface.implements(checker, interfaces.IAstroidChecker): |
| walker.add_checker(checker) |
| # build ast and check modules or packages |
| for descr in self.expand_files(files_or_modules): |
| modname, filepath = descr['name'], descr['path'] |
| if not descr['isarg'] and not self.should_analyze_file(modname, filepath): |
| continue |
| if self.config.files_output: |
| reportfile = 'pylint_%s.%s' % (modname, self.reporter.extension) |
| self.reporter.set_output(open(reportfile, 'w')) |
| self.set_current_module(modname, filepath) |
| # get the module representation |
| ast_node = self.get_ast(filepath, modname) |
| if ast_node is None: |
| continue |
| # XXX to be correct we need to keep module_msgs_state for every |
| # analyzed module (the problem stands with localized messages which |
| # are only detected in the .close step) |
| self.file_state = utils.FileState(descr['basename']) |
| self._ignore_file = False |
| # fix the current file (if the source file was not available or |
| # if it's actually a c extension) |
| self.current_file = ast_node.file # pylint: disable=maybe-no-member |
| self.check_astroid_module(ast_node, walker, rawcheckers, tokencheckers) |
| # warn about spurious inline messages handling |
| for msgid, line, args in self.file_state.iter_spurious_suppression_messages(self.msgs_store): |
| self.add_message(msgid, line, None, args) |
| # notify global end |
| self.stats['statement'] = walker.nbstatements |
| checkers.reverse() |
| for checker in checkers: |
| checker.close() |
| |
| def expand_files(self, modules): |
| """get modules and errors from a list of modules and handle errors |
| """ |
| result, errors = utils.expand_modules(modules, self.config.black_list) |
| for error in errors: |
| message = modname = error["mod"] |
| key = error["key"] |
| self.set_current_module(modname) |
| if key == "fatal": |
| message = str(error["ex"]).replace(os.getcwd() + os.sep, '') |
| self.add_message(key, args=message) |
| return result |
| |
| def set_current_module(self, modname, filepath=None): |
| """set the name of the currently analyzed module and |
| init statistics for it |
| """ |
| if not modname and filepath is None: |
| return |
| self.reporter.on_set_current_module(modname, filepath) |
| self.current_name = modname |
| self.current_file = filepath or modname |
| self.stats['by_module'][modname] = {} |
| self.stats['by_module'][modname]['statement'] = 0 |
| for msg_cat in six.itervalues(utils.MSG_TYPES): |
| self.stats['by_module'][modname][msg_cat] = 0 |
| |
| def get_ast(self, filepath, modname): |
| """return a ast(roid) representation for a module""" |
| try: |
| return MANAGER.ast_from_file(filepath, modname, source=True) |
| except SyntaxError as ex: |
| self.add_message('syntax-error', line=ex.lineno, args=ex.msg) |
| except astroid.AstroidBuildingException as ex: |
| self.add_message('parse-error', args=ex) |
| except Exception as ex: # pylint: disable=broad-except |
| import traceback |
| traceback.print_exc() |
| self.add_message('astroid-error', args=(ex.__class__, ex)) |
| |
| def check_astroid_module(self, ast_node, walker, |
| rawcheckers, tokencheckers): |
| """Check a module from its astroid representation.""" |
| try: |
| tokens = utils.tokenize_module(ast_node) |
| except tokenize.TokenError as ex: |
| self.add_message('syntax-error', line=ex.args[1][0], args=ex.args[0]) |
| return |
| |
| if not ast_node.pure_python: |
| self.add_message('raw-checker-failed', args=ast_node.name) |
| else: |
| #assert astroid.file.endswith('.py') |
| # invoke ITokenChecker interface on self to fetch module/block |
| # level options |
| self.process_tokens(tokens) |
| if self._ignore_file: |
| return False |
| # walk ast to collect line numbers |
| self.file_state.collect_block_lines(self.msgs_store, ast_node) |
| # run raw and tokens checkers |
| for checker in rawcheckers: |
| checker.process_module(ast_node) |
| for checker in tokencheckers: |
| checker.process_tokens(tokens) |
| # generate events to astroid checkers |
| walker.walk(ast_node) |
| return True |
| |
| # IAstroidChecker interface ################################################# |
| |
| def open(self): |
| """initialize counters""" |
| self.stats = {'by_module' : {}, |
| 'by_msg' : {}, |
| } |
| MANAGER.always_load_extensions = self.config.unsafe_load_any_extension |
| MANAGER.extension_package_whitelist.update( |
| self.config.extension_pkg_whitelist) |
| for msg_cat in six.itervalues(utils.MSG_TYPES): |
| self.stats[msg_cat] = 0 |
| |
| def generate_reports(self): |
| """close the whole package /module, it's time to make reports ! |
| |
| if persistent run, pickle results for later comparison |
| """ |
| if self.file_state.base_name is not None: |
| # load previous results if any |
| previous_stats = config.load_results(self.file_state.base_name) |
| # XXX code below needs refactoring to be more reporter agnostic |
| self.reporter.on_close(self.stats, previous_stats) |
| if self.config.reports: |
| sect = self.make_reports(self.stats, previous_stats) |
| if self.config.files_output: |
| filename = 'pylint_global.' + self.reporter.extension |
| self.reporter.set_output(open(filename, 'w')) |
| else: |
| sect = ureports.Section() |
| if self.config.reports or self.config.output_format == 'html': |
| self.reporter.display_results(sect) |
| # save results if persistent run |
| if self.config.persistent: |
| config.save_results(self.stats, self.file_state.base_name) |
| else: |
| if self.config.output_format == 'html': |
| # No output will be emitted for the html |
| # reporter if the file doesn't exist, so emit |
| # the results here. |
| self.reporter.display_results(ureports.Section()) |
| self.reporter.on_close(self.stats, {}) |
| |
| # specific reports ######################################################## |
| |
| def report_evaluation(self, sect, stats, previous_stats): |
| """make the global evaluation report""" |
| # check with at least check 1 statements (usually 0 when there is a |
| # syntax error preventing pylint from further processing) |
| if stats['statement'] == 0: |
| raise utils.EmptyReport() |
| # get a global note for the code |
| evaluation = self.config.evaluation |
| try: |
| note = eval(evaluation, {}, self.stats) # pylint: disable=eval-used |
| except Exception as ex: # pylint: disable=broad-except |
| msg = 'An exception occurred while rating: %s' % ex |
| else: |
| stats['global_note'] = note |
| msg = 'Your code has been rated at %.2f/10' % note |
| pnote = previous_stats.get('global_note') |
| if pnote is not None: |
| msg += ' (previous run: %.2f/10, %+.2f)' % (pnote, note - pnote) |
| if self.config.comment: |
| msg = '%s\n%s' % (msg, config.get_note_message(note)) |
| sect.append(ureports.Text(msg)) |
| |
| # some reporting functions #################################################### |
| |
| def report_total_messages_stats(sect, stats, previous_stats): |
| """make total errors / warnings report""" |
| lines = ['type', 'number', 'previous', 'difference'] |
| lines += checkers.table_lines_from_stats(stats, previous_stats, |
| ('convention', 'refactor', |
| 'warning', 'error')) |
| sect.append(ureports.Table(children=lines, cols=4, rheaders=1)) |
| |
| def report_messages_stats(sect, stats, _): |
| """make messages type report""" |
| if not stats['by_msg']: |
| # don't print this report when we didn't detected any errors |
| raise utils.EmptyReport() |
| in_order = sorted([(value, msg_id) |
| for msg_id, value in six.iteritems(stats['by_msg']) |
| if not msg_id.startswith('I')]) |
| in_order.reverse() |
| lines = ('message id', 'occurrences') |
| for value, msg_id in in_order: |
| lines += (msg_id, str(value)) |
| sect.append(ureports.Table(children=lines, cols=2, rheaders=1)) |
| |
| def report_messages_by_module_stats(sect, stats, _): |
| """make errors / warnings by modules report""" |
| if len(stats['by_module']) == 1: |
| # don't print this report when we are analysing a single module |
| raise utils.EmptyReport() |
| by_mod = collections.defaultdict(dict) |
| for m_type in ('fatal', 'error', 'warning', 'refactor', 'convention'): |
| total = stats[m_type] |
| for module in six.iterkeys(stats['by_module']): |
| mod_total = stats['by_module'][module][m_type] |
| if total == 0: |
| percent = 0 |
| else: |
| percent = float((mod_total)*100) / total |
| by_mod[module][m_type] = percent |
| sorted_result = [] |
| for module, mod_info in six.iteritems(by_mod): |
| sorted_result.append((mod_info['error'], |
| mod_info['warning'], |
| mod_info['refactor'], |
| mod_info['convention'], |
| module)) |
| sorted_result.sort() |
| sorted_result.reverse() |
| lines = ['module', 'error', 'warning', 'refactor', 'convention'] |
| for line in sorted_result: |
| # Don't report clean modules. |
| if all(entry == 0 for entry in line[:-1]): |
| continue |
| lines.append(line[-1]) |
| for val in line[:-1]: |
| lines.append('%.2f' % val) |
| if len(lines) == 5: |
| raise utils.EmptyReport() |
| sect.append(ureports.Table(children=lines, cols=5, rheaders=1)) |
| |
| |
| # utilities ################################################################### |
| |
| |
| class ArgumentPreprocessingError(Exception): |
| """Raised if an error occurs during argument preprocessing.""" |
| |
| |
| def preprocess_options(args, search_for): |
| """look for some options (keys of <search_for>) which have to be processed |
| before others |
| |
| values of <search_for> are callback functions to call when the option is |
| found |
| """ |
| i = 0 |
| while i < len(args): |
| arg = args[i] |
| if arg.startswith('--'): |
| try: |
| option, val = arg[2:].split('=', 1) |
| except ValueError: |
| option, val = arg[2:], None |
| try: |
| cb, takearg = search_for[option] |
| except KeyError: |
| i += 1 |
| else: |
| del args[i] |
| if takearg and val is None: |
| if i >= len(args) or args[i].startswith('-'): |
| msg = 'Option %s expects a value' % option |
| raise ArgumentPreprocessingError(msg) |
| val = args[i] |
| del args[i] |
| elif not takearg and val is not None: |
| msg = "Option %s doesn't expects a value" % option |
| raise ArgumentPreprocessingError(msg) |
| cb(option, val) |
| else: |
| i += 1 |
| |
| |
| @contextlib.contextmanager |
| def fix_import_path(args): |
| """Prepare sys.path for running the linter checks. |
| |
| Within this context, each of the given arguments is importable. |
| Paths are added to sys.path in corresponding order to the arguments. |
| We avoid adding duplicate directories to sys.path. |
| `sys.path` is reset to its original value upon exitign this context. |
| """ |
| orig = list(sys.path) |
| changes = [] |
| for arg in args: |
| path = _get_python_path(arg) |
| if path in changes: |
| continue |
| else: |
| changes.append(path) |
| sys.path[:] = changes + sys.path |
| try: |
| yield |
| finally: |
| sys.path[:] = orig |
| |
| |
| class Run(object): |
| """helper class to use as main for pylint : |
| |
| run(*sys.argv[1:]) |
| """ |
| LinterClass = PyLinter |
| option_groups = ( |
| ('Commands', 'Options which are actually commands. Options in this \ |
| group are mutually exclusive.'), |
| ) |
| |
| def __init__(self, args, reporter=None, exit=True): |
| self._rcfile = None |
| self._plugins = [] |
| try: |
| preprocess_options(args, { |
| # option: (callback, takearg) |
| 'init-hook': (cb_init_hook, True), |
| 'rcfile': (self.cb_set_rcfile, True), |
| 'load-plugins': (self.cb_add_plugins, True), |
| }) |
| except ArgumentPreprocessingError as ex: |
| print(ex, file=sys.stderr) |
| sys.exit(32) |
| |
| self.linter = linter = self.LinterClass(( |
| ('rcfile', |
| {'action' : 'callback', 'callback' : lambda *args: 1, |
| 'type': 'string', 'metavar': '<file>', |
| 'help' : 'Specify a configuration file.'}), |
| |
| ('init-hook', |
| {'action' : 'callback', 'callback' : lambda *args: 1, |
| 'type' : 'string', 'metavar': '<code>', |
| 'level': 1, |
| 'help' : 'Python code to execute, usually for sys.path ' |
| 'manipulation such as pygtk.require().'}), |
| |
| ('help-msg', |
| {'action' : 'callback', 'type' : 'string', 'metavar': '<msg-id>', |
| 'callback' : self.cb_help_message, |
| 'group': 'Commands', |
| 'help' : 'Display a help message for the given message id and ' |
| 'exit. The value may be a comma separated list of message ids.'}), |
| |
| ('list-msgs', |
| {'action' : 'callback', 'metavar': '<msg-id>', |
| 'callback' : self.cb_list_messages, |
| 'group': 'Commands', 'level': 1, |
| 'help' : "Generate pylint's messages."}), |
| |
| ('list-conf-levels', |
| {'action' : 'callback', |
| 'callback' : cb_list_confidence_levels, |
| 'group': 'Commands', 'level': 1, |
| 'help' : "Generate pylint's messages."}), |
| |
| ('full-documentation', |
| {'action' : 'callback', 'metavar': '<msg-id>', |
| 'callback' : self.cb_full_documentation, |
| 'group': 'Commands', 'level': 1, |
| 'help' : "Generate pylint's full documentation."}), |
| |
| ('generate-rcfile', |
| {'action' : 'callback', 'callback' : self.cb_generate_config, |
| 'group': 'Commands', |
| 'help' : 'Generate a sample configuration file according to ' |
| 'the current configuration. You can put other options ' |
| 'before this one to get them in the generated ' |
| 'configuration.'}), |
| |
| ('generate-man', |
| {'action' : 'callback', 'callback' : self.cb_generate_manpage, |
| 'group': 'Commands', |
| 'help' : "Generate pylint's man page.", 'hide': True}), |
| |
| ('errors-only', |
| {'action' : 'callback', 'callback' : self.cb_error_mode, |
| 'short': 'E', |
| 'help' : 'In error mode, checkers without error messages are ' |
| 'disabled and for others, only the ERROR messages are ' |
| 'displayed, and no reports are done by default'''}), |
| |
| ('py3k', |
| {'action' : 'callback', 'callback' : self.cb_python3_porting_mode, |
| 'help' : 'In Python 3 porting mode, all checkers will be ' |
| 'disabled and only messages emitted by the porting ' |
| 'checker will be displayed'}), |
| |
| ('profile', |
| {'type' : 'yn', 'metavar' : '<y_or_n>', |
| 'default': False, 'hide': True, |
| 'help' : 'Profiled execution.'}), |
| |
| ), option_groups=self.option_groups, pylintrc=self._rcfile) |
| # register standard checkers |
| linter.load_default_plugins() |
| # load command line plugins |
| linter.load_plugin_modules(self._plugins) |
| # add some help section |
| linter.add_help_section('Environment variables', config.ENV_HELP, level=1) |
| # pylint: disable=bad-continuation |
| linter.add_help_section('Output', |
| 'Using the default text output, the message format is : \n' |
| ' \n' |
| ' MESSAGE_TYPE: LINE_NUM:[OBJECT:] MESSAGE \n' |
| ' \n' |
| 'There are 5 kind of message types : \n' |
| ' * (C) convention, for programming standard violation \n' |
| ' * (R) refactor, for bad code smell \n' |
| ' * (W) warning, for python specific problems \n' |
| ' * (E) error, for probable bugs in the code \n' |
| ' * (F) fatal, if an error occurred which prevented pylint from doing further\n' |
| 'processing.\n' |
| , level=1) |
| linter.add_help_section('Output status code', |
| 'Pylint should leave with following status code: \n' |
| ' * 0 if everything went fine \n' |
| ' * 1 if a fatal message was issued \n' |
| ' * 2 if an error message was issued \n' |
| ' * 4 if a warning message was issued \n' |
| ' * 8 if a refactor message was issued \n' |
| ' * 16 if a convention message was issued \n' |
| ' * 32 on usage error \n' |
| ' \n' |
| 'status 1 to 16 will be bit-ORed so you can know which different categories has\n' |
| 'been issued by analysing pylint output status code\n', |
| level=1) |
| # read configuration |
| linter.disable('pointless-except') |
| linter.disable('suppressed-message') |
| linter.disable('useless-suppression') |
| linter.read_config_file() |
| config_parser = linter.cfgfile_parser |
| # run init hook, if present, before loading plugins |
| if config_parser.has_option('MASTER', 'init-hook'): |
| cb_init_hook('init-hook', |
| textutils.unquote(config_parser.get('MASTER', |
| 'init-hook'))) |
| # is there some additional plugins in the file configuration, in |
| if config_parser.has_option('MASTER', 'load-plugins'): |
| plugins = textutils.splitstrip( |
| config_parser.get('MASTER', 'load-plugins')) |
| linter.load_plugin_modules(plugins) |
| # now we can load file config and command line, plugins (which can |
| # provide options) have been registered |
| linter.load_config_file() |
| if reporter: |
| # if a custom reporter is provided as argument, it may be overridden |
| # by file parameters, so re-set it here, but before command line |
| # parsing so it's still overrideable by command line option |
| linter.set_reporter(reporter) |
| try: |
| args = linter.load_command_line_configuration(args) |
| except SystemExit as exc: |
| if exc.code == 2: # bad options |
| exc.code = 32 |
| raise |
| if not args: |
| print(linter.help()) |
| sys.exit(32) |
| |
| if linter.config.jobs < 0: |
| print("Jobs number (%d) should be greater than 0" |
| % linter.config.jobs, file=sys.stderr) |
| sys.exit(32) |
| if linter.config.jobs > 1 or linter.config.jobs == 0: |
| if multiprocessing is None: |
| print("Multiprocessing library is missing, " |
| "fallback to single process", file=sys.stderr) |
| linter.set_option("jobs", 1) |
| else: |
| if linter.config.jobs == 0: |
| linter.config.jobs = multiprocessing.cpu_count() |
| |
| # insert current working directory to the python path to have a correct |
| # behaviour |
| if self.linter.config.profile: |
| with fix_import_path(args): |
| print('** profiled run', file=sys.stderr) |
| import cProfile, pstats |
| cProfile.runctx('linter.check(%r)' % args, globals(), locals(), |
| 'stones.prof') |
| data = pstats.Stats('stones.prof') |
| data.strip_dirs() |
| data.sort_stats('time', 'calls') |
| data.print_stats(30) |
| else: |
| linter.check(args) |
| linter.generate_reports() |
| if exit: |
| sys.exit(self.linter.msg_status) |
| |
| def cb_set_rcfile(self, name, value): |
| """callback for option preprocessing (i.e. before option parsing)""" |
| self._rcfile = value |
| |
| def cb_add_plugins(self, name, value): |
| """callback for option preprocessing (i.e. before option parsing)""" |
| self._plugins.extend(textutils.splitstrip(value)) |
| |
| def cb_error_mode(self, *args, **kwargs): |
| """error mode: |
| * disable all but error messages |
| * disable the 'miscellaneous' checker which can be safely deactivated in |
| debug |
| * disable reports |
| * do not save execution information |
| """ |
| self.linter.error_mode() |
| |
| def cb_generate_config(self, *args, **kwargs): |
| """optik callback for sample config file generation""" |
| self.linter.generate_config(skipsections=('COMMANDS',)) |
| sys.exit(0) |
| |
| def cb_generate_manpage(self, *args, **kwargs): |
| """optik callback for sample config file generation""" |
| from pylint import __pkginfo__ |
| self.linter.generate_manpage(__pkginfo__) |
| sys.exit(0) |
| |
| def cb_help_message(self, option, optname, value, parser): |
| """optik callback for printing some help about a particular message""" |
| self.linter.msgs_store.help_message(textutils.splitstrip(value)) |
| sys.exit(0) |
| |
| def cb_full_documentation(self, option, optname, value, parser): |
| """optik callback for printing full documentation""" |
| self.linter.print_full_documentation() |
| sys.exit(0) |
| |
| def cb_list_messages(self, option, optname, value, parser): # FIXME |
| """optik callback for printing available messages""" |
| self.linter.msgs_store.list_messages() |
| sys.exit(0) |
| |
| def cb_python3_porting_mode(self, *args, **kwargs): |
| """Activate only the python3 porting checker.""" |
| self.linter.disable('all') |
| self.linter.enable('python3') |
| |
| |
| def cb_list_confidence_levels(option, optname, value, parser): |
| for level in interfaces.CONFIDENCE_LEVELS: |
| print('%-18s: %s' % level) |
| sys.exit(0) |
| |
| def cb_init_hook(optname, value): |
| """exec arbitrary code to set sys.path for instance""" |
| exec(value) # pylint: disable=exec-used |
| |
| |
| if __name__ == '__main__': |
| Run(sys.argv[1:]) |
