| |
| """ |
| yaml.py |
| |
| Lexer for YAML, a human-friendly data serialization language |
| (http://yaml.org/). |
| |
| Written by Kirill Simonov <xi@resolvent.net>. |
| |
| License: Whatever suitable for inclusion into the Pygments package. |
| """ |
| |
| from pygments.lexer import \ |
| ExtendedRegexLexer, LexerContext, include, bygroups |
| from pygments.token import \ |
| Text, Comment, Punctuation, Name, Literal |
| |
| __all__ = ['YAMLLexer'] |
| |
| |
| class YAMLLexerContext(LexerContext): |
| """Indentation context for the YAML lexer.""" |
| |
| def __init__(self, *args, **kwds): |
| super(YAMLLexerContext, self).__init__(*args, **kwds) |
| self.indent_stack = [] |
| self.indent = -1 |
| self.next_indent = 0 |
| self.block_scalar_indent = None |
| |
| |
| def something(TokenClass): |
| """Do not produce empty tokens.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if not text: |
| return |
| yield match.start(), TokenClass, text |
| context.pos = match.end() |
| return callback |
| |
| def reset_indent(TokenClass): |
| """Reset the indentation levels.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| context.indent_stack = [] |
| context.indent = -1 |
| context.next_indent = 0 |
| context.block_scalar_indent = None |
| yield match.start(), TokenClass, text |
| context.pos = match.end() |
| return callback |
| |
| def save_indent(TokenClass, start=False): |
| """Save a possible indentation level.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| extra = '' |
| if start: |
| context.next_indent = len(text) |
| if context.next_indent < context.indent: |
| while context.next_indent < context.indent: |
| context.indent = context.indent_stack.pop() |
| if context.next_indent > context.indent: |
| extra = text[context.indent:] |
| text = text[:context.indent] |
| else: |
| context.next_indent += len(text) |
| if text: |
| yield match.start(), TokenClass, text |
| if extra: |
| yield match.start()+len(text), TokenClass.Error, extra |
| context.pos = match.end() |
| return callback |
| |
| def set_indent(TokenClass, implicit=False): |
| """Set the previously saved indentation level.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if context.indent < context.next_indent: |
| context.indent_stack.append(context.indent) |
| context.indent = context.next_indent |
| if not implicit: |
| context.next_indent += len(text) |
| yield match.start(), TokenClass, text |
| context.pos = match.end() |
| return callback |
| |
| def set_block_scalar_indent(TokenClass): |
| """Set an explicit indentation level for a block scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| context.block_scalar_indent = None |
| if not text: |
| return |
| increment = match.group(1) |
| if increment: |
| current_indent = max(context.indent, 0) |
| increment = int(increment) |
| context.block_scalar_indent = current_indent + increment |
| if text: |
| yield match.start(), TokenClass, text |
| context.pos = match.end() |
| return callback |
| |
| def parse_block_scalar_empty_line(IndentTokenClass, ContentTokenClass): |
| """Process an empty line in a block scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if (context.block_scalar_indent is None or |
| len(text) <= context.block_scalar_indent): |
| if text: |
| yield match.start(), IndentTokenClass, text |
| else: |
| indentation = text[:context.block_scalar_indent] |
| content = text[context.block_scalar_indent:] |
| yield match.start(), IndentTokenClass, indentation |
| yield (match.start()+context.block_scalar_indent, |
| ContentTokenClass, content) |
| context.pos = match.end() |
| return callback |
| |
| def parse_block_scalar_indent(TokenClass): |
| """Process indentation spaces in a block scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if context.block_scalar_indent is None: |
| if len(text) <= max(context.indent, 0): |
| context.stack.pop() |
| context.stack.pop() |
| return |
| context.block_scalar_indent = len(text) |
| else: |
| if len(text) < context.block_scalar_indent: |
| context.stack.pop() |
| context.stack.pop() |
| return |
| if text: |
| yield match.start(), TokenClass, text |
| context.pos = match.end() |
| return callback |
| |
| def parse_plain_scalar_indent(TokenClass): |
| """Process indentation spaces in a plain scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if len(text) <= context.indent: |
| context.stack.pop() |
| context.stack.pop() |
| return |
| if text: |
| yield match.start(), TokenClass, text |
| context.pos = match.end() |
| return callback |
| |
| |
| class YAMLLexer(ExtendedRegexLexer): |
| """Lexer for the YAML language.""" |
| |
| name = 'YAML' |
| aliases = ['yaml'] |
| filenames = ['*.yaml', '*.yml'] |
| mimetypes = ['text/x-yaml'] |
| |
| tokens = { |
| |
| # the root rules |
| 'root': [ |
| # ignored whitespaces |
| (r'[ ]+(?=#|$)', Text.Blank), |
| # line breaks |
| (r'\n+', Text.Break), |
| # a comment |
| (r'#[^\n]*', Comment.Single), |
| # the '%YAML' directive |
| (r'^%YAML(?=[ ]|$)', reset_indent(Name.Directive), |
| 'yaml-directive'), |
| # the %TAG directive |
| (r'^%TAG(?=[ ]|$)', reset_indent(Name.Directive), |
| 'tag-directive'), |
| # document start and document end indicators |
| (r'^(?:---|\.\.\.)(?=[ ]|$)', |
| reset_indent(Punctuation.Document), 'block-line'), |
| # indentation spaces |
| (r'[ ]*(?![ \t\n\r\f\v]|$)', |
| save_indent(Text.Indent, start=True), |
| ('block-line', 'indentation')), |
| ], |
| |
| # trailing whitespaces after directives or a block scalar indicator |
| 'ignored-line': [ |
| # ignored whitespaces |
| (r'[ ]+(?=#|$)', Text.Blank), |
| # a comment |
| (r'#[^\n]*', Comment.Single), |
| # line break |
| (r'\n', Text.Break, '#pop:2'), |
| ], |
| |
| # the %YAML directive |
| 'yaml-directive': [ |
| # the version number |
| (r'([ ]+)([0-9]+\.[0-9]+)', |
| bygroups(Text.Blank, Literal.Version), 'ignored-line'), |
| ], |
| |
| # the %YAG directive |
| 'tag-directive': [ |
| # a tag handle and the corresponding prefix |
| (r'([ ]+)(!|![0-9A-Za-z_-]*!)' |
| r'([ ]+)(!|!?[0-9A-Za-z;/?:@&=+$,_.!~*\'()\[\]%-]+)', |
| bygroups(Text.Blank, Name.Type, Text.Blank, Name.Type), |
| 'ignored-line'), |
| ], |
| |
| # block scalar indicators and indentation spaces |
| 'indentation': [ |
| # trailing whitespaces are ignored |
| (r'[ ]*$', something(Text.Blank), '#pop:2'), |
| # whitespaces preceeding block collection indicators |
| (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Text.Indent)), |
| # block collection indicators |
| (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)), |
| # the beginning a block line |
| (r'[ ]*', save_indent(Text.Indent), '#pop'), |
| ], |
| |
| # an indented line in the block context |
| 'block-line': [ |
| # the line end |
| (r'[ ]*(?=#|$)', something(Text.Blank), '#pop'), |
| # whitespaces separating tokens |
| (r'[ ]+', Text.Blank), |
| # tags, anchors and aliases, |
| include('descriptors'), |
| # block collections and scalars |
| include('block-nodes'), |
| # flow collections and quoted scalars |
| include('flow-nodes'), |
| # a plain scalar |
| (r'(?=[^ \t\n\r\f\v?:,\[\]{}#&*!|>\'"%@`-]|[?:-][^ \t\n\r\f\v])', |
| something(Literal.Scalar.Plain), |
| 'plain-scalar-in-block-context'), |
| ], |
| |
| # tags, anchors, aliases |
| 'descriptors' : [ |
| # a full-form tag |
| (r'!<[0-9A-Za-z;/?:@&=+$,_.!~*\'()\[\]%-]+>', Name.Type), |
| # a tag in the form '!', '!suffix' or '!handle!suffix' |
| (r'!(?:[0-9A-Za-z_-]+)?' |
| r'(?:![0-9A-Za-z;/?:@&=+$,_.!~*\'()\[\]%-]+)?', Name.Type), |
| # an anchor |
| (r'&[0-9A-Za-z_-]+', Name.Anchor), |
| # an alias |
| (r'\*[0-9A-Za-z_-]+', Name.Alias), |
| ], |
| |
| # block collections and scalars |
| 'block-nodes': [ |
| # implicit key |
| (r':(?=[ ]|$)', set_indent(Punctuation.Indicator, implicit=True)), |
| # literal and folded scalars |
| (r'[|>]', Punctuation.Indicator, |
| ('block-scalar-content', 'block-scalar-header')), |
| ], |
| |
| # flow collections and quoted scalars |
| 'flow-nodes': [ |
| # a flow sequence |
| (r'\[', Punctuation.Indicator, 'flow-sequence'), |
| # a flow mapping |
| (r'\{', Punctuation.Indicator, 'flow-mapping'), |
| # a single-quoted scalar |
| (r'\'', Literal.Scalar.Flow.Quote, 'single-quoted-scalar'), |
| # a double-quoted scalar |
| (r'\"', Literal.Scalar.Flow.Quote, 'double-quoted-scalar'), |
| ], |
| |
| # the content of a flow collection |
| 'flow-collection': [ |
| # whitespaces |
| (r'[ ]+', Text.Blank), |
| # line breaks |
| (r'\n+', Text.Break), |
| # a comment |
| (r'#[^\n]*', Comment.Single), |
| # simple indicators |
| (r'[?:,]', Punctuation.Indicator), |
| # tags, anchors and aliases |
| include('descriptors'), |
| # nested collections and quoted scalars |
| include('flow-nodes'), |
| # a plain scalar |
| (r'(?=[^ \t\n\r\f\v?:,\[\]{}#&*!|>\'"%@`])', |
| something(Literal.Scalar.Plain), |
| 'plain-scalar-in-flow-context'), |
| ], |
| |
| # a flow sequence indicated by '[' and ']' |
| 'flow-sequence': [ |
| # include flow collection rules |
| include('flow-collection'), |
| # the closing indicator |
| (r'\]', Punctuation.Indicator, '#pop'), |
| ], |
| |
| # a flow mapping indicated by '{' and '}' |
| 'flow-mapping': [ |
| # include flow collection rules |
| include('flow-collection'), |
| # the closing indicator |
| (r'\}', Punctuation.Indicator, '#pop'), |
| ], |
| |
| # block scalar lines |
| 'block-scalar-content': [ |
| # line break |
| (r'\n', Text.Break), |
| # empty line |
| (r'^[ ]+$', |
| parse_block_scalar_empty_line(Text.Indent, |
| Literal.Scalar.Block)), |
| # indentation spaces (we may leave the state here) |
| (r'^[ ]*', parse_block_scalar_indent(Text.Indent)), |
| # line content |
| (r'[^\n\r\f\v]+', Literal.Scalar.Block), |
| ], |
| |
| # the content of a literal or folded scalar |
| 'block-scalar-header': [ |
| # indentation indicator followed by chomping flag |
| (r'([1-9])?[+-]?(?=[ ]|$)', |
| set_block_scalar_indent(Punctuation.Indicator), |
| 'ignored-line'), |
| # chomping flag followed by indentation indicator |
| (r'[+-]?([1-9])?(?=[ ]|$)', |
| set_block_scalar_indent(Punctuation.Indicator), |
| 'ignored-line'), |
| ], |
| |
| # ignored and regular whitespaces in quoted scalars |
| 'quoted-scalar-whitespaces': [ |
| # leading and trailing whitespaces are ignored |
| (r'^[ ]+|[ ]+$', Text.Blank), |
| # line breaks are ignored |
| (r'\n+', Text.Break), |
| # other whitespaces are a part of the value |
| (r'[ ]+', Literal.Scalar.Flow), |
| ], |
| |
| # single-quoted scalars |
| 'single-quoted-scalar': [ |
| # include whitespace and line break rules |
| include('quoted-scalar-whitespaces'), |
| # escaping of the quote character |
| (r'\'\'', Literal.Scalar.Flow.Escape), |
| # regular non-whitespace characters |
| (r'[^ \t\n\r\f\v\']+', Literal.Scalar.Flow), |
| # the closing quote |
| (r'\'', Literal.Scalar.Flow.Quote, '#pop'), |
| ], |
| |
| # double-quoted scalars |
| 'double-quoted-scalar': [ |
| # include whitespace and line break rules |
| include('quoted-scalar-whitespaces'), |
| # escaping of special characters |
| (r'\\[0abt\tn\nvfre "\\N_LP]', Literal.Scalar.Flow.Escape), |
| # escape codes |
| (r'\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})', |
| Literal.Scalar.Flow.Escape), |
| # regular non-whitespace characters |
| (r'[^ \t\n\r\f\v\"\\]+', Literal.Scalar.Flow), |
| # the closing quote |
| (r'"', Literal.Scalar.Flow.Quote, '#pop'), |
| ], |
| |
| # the beginning of a new line while scanning a plain scalar |
| 'plain-scalar-in-block-context-new-line': [ |
| # empty lines |
| (r'^[ ]+$', Text.Blank), |
| # line breaks |
| (r'\n+', Text.Break), |
| # document start and document end indicators |
| (r'^(?=---|\.\.\.)', something(Punctuation.Document), '#pop:3'), |
| # indentation spaces (we may leave the block line state here) |
| (r'^[ ]*', parse_plain_scalar_indent(Text.Indent), '#pop'), |
| ], |
| |
| # a plain scalar in the block context |
| 'plain-scalar-in-block-context': [ |
| # the scalar ends with the ':' indicator |
| (r'[ ]*(?=:[ ]|:$)', something(Text.Blank), '#pop'), |
| # the scalar ends with whitespaces followed by a comment |
| (r'[ ]+(?=#)', Text.Blank, '#pop'), |
| # trailing whitespaces are ignored |
| (r'[ ]+$', Text.Blank), |
| # line breaks are ignored |
| (r'\n+', Text.Break, 'plain-scalar-in-block-context-new-line'), |
| # other whitespaces are a part of the value |
| (r'[ ]+', Literal.Scalar.Plain), |
| # regular non-whitespace characters |
| (r'(?::(?![ \t\n\r\f\v])|[^ \t\n\r\f\v:])+', |
| Literal.Scalar.Plain), |
| ], |
| |
| # a plain scalar is the flow context |
| 'plain-scalar-in-flow-context': [ |
| # the scalar ends with an indicator character |
| (r'[ ]*(?=[,:?\[\]{}])', something(Text.Blank), '#pop'), |
| # the scalar ends with a comment |
| (r'[ ]+(?=#)', Text.Blank, '#pop'), |
| # leading and trailing whitespaces are ignored |
| (r'^[ ]+|[ ]+$', Text.Blank), |
| # line breaks are ignored |
| (r'\n+', Text.Break), |
| # other whitespaces are a part of the value |
| (r'[ ]+', Literal.Scalar.Plain), |
| # regular non-whitespace characters |
| (r'[^ \t\n\r\f\v,:?\[\]{}]+', Literal.Scalar.Plain), |
| ], |
| |
| } |
| |
| def get_tokens_unprocessed(self, text=None, context=None): |
| if context is None: |
| context = YAMLLexerContext(text, 0) |
| return super(YAMLLexer, self).get_tokens_unprocessed(text, context) |
| |
| |
