token.py
195 lines
| 5.5 KiB
| text/x-python
|
PythonLexer
Jeandet Alexis
|
r0 | # -*- coding: utf-8 -*- | ||
| """ | ||||
| pygments.token | ||||
| ~~~~~~~~~~~~~~ | ||||
| Basic token types and the standard tokens. | ||||
| :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. | ||||
| :license: BSD, see LICENSE for details. | ||||
| """ | ||||
| class _TokenType(tuple): | ||||
| parent = None | ||||
| def split(self): | ||||
| buf = [] | ||||
| node = self | ||||
| while node is not None: | ||||
| buf.append(node) | ||||
| node = node.parent | ||||
| buf.reverse() | ||||
| return buf | ||||
| def __init__(self, *args): | ||||
| # no need to call super.__init__ | ||||
| self.subtypes = set() | ||||
| def __contains__(self, val): | ||||
| return self is val or ( | ||||
| type(val) is self.__class__ and | ||||
| val[:len(self)] == self | ||||
| ) | ||||
| def __getattr__(self, val): | ||||
| if not val or not val[0].isupper(): | ||||
| return tuple.__getattribute__(self, val) | ||||
| new = _TokenType(self + (val,)) | ||||
| setattr(self, val, new) | ||||
| self.subtypes.add(new) | ||||
| new.parent = self | ||||
| return new | ||||
| def __repr__(self): | ||||
| return 'Token' + (self and '.' or '') + '.'.join(self) | ||||
| Token = _TokenType() | ||||
| # Special token types | ||||
| Text = Token.Text | ||||
| Whitespace = Text.Whitespace | ||||
| Error = Token.Error | ||||
| # Text that doesn't belong to this lexer (e.g. HTML in PHP) | ||||
| Other = Token.Other | ||||
| # Common token types for source code | ||||
| Keyword = Token.Keyword | ||||
| Name = Token.Name | ||||
| Literal = Token.Literal | ||||
| String = Literal.String | ||||
| Number = Literal.Number | ||||
| Punctuation = Token.Punctuation | ||||
| Operator = Token.Operator | ||||
| Comment = Token.Comment | ||||
| # Generic types for non-source code | ||||
| Generic = Token.Generic | ||||
| # String and some others are not direct childs of Token. | ||||
| # alias them: | ||||
| Token.Token = Token | ||||
| Token.String = String | ||||
| Token.Number = Number | ||||
| def is_token_subtype(ttype, other): | ||||
| """ | ||||
| Return True if ``ttype`` is a subtype of ``other``. | ||||
| exists for backwards compatibility. use ``ttype in other`` now. | ||||
| """ | ||||
| return ttype in other | ||||
| def string_to_tokentype(s): | ||||
| """ | ||||
| Convert a string into a token type:: | ||||
| >>> string_to_token('String.Double') | ||||
| Token.Literal.String.Double | ||||
| >>> string_to_token('Token.Literal.Number') | ||||
| Token.Literal.Number | ||||
| >>> string_to_token('') | ||||
| Token | ||||
| Tokens that are already tokens are returned unchanged: | ||||
| >>> string_to_token(String) | ||||
| Token.Literal.String | ||||
| """ | ||||
| if isinstance(s, _TokenType): | ||||
| return s | ||||
| if not s: | ||||
| return Token | ||||
| node = Token | ||||
| for item in s.split('.'): | ||||
| node = getattr(node, item) | ||||
| return node | ||||
| # Map standard token types to short names, used in CSS class naming. | ||||
| # If you add a new item, please be sure to run this file to perform | ||||
| # a consistency check for duplicate values. | ||||
| STANDARD_TYPES = { | ||||
| Token: '', | ||||
| Text: '', | ||||
| Whitespace: 'w', | ||||
| Error: 'err', | ||||
| Other: 'x', | ||||
| Keyword: 'k', | ||||
| Keyword.Constant: 'kc', | ||||
| Keyword.Declaration: 'kd', | ||||
| Keyword.Namespace: 'kn', | ||||
| Keyword.Pseudo: 'kp', | ||||
| Keyword.Reserved: 'kr', | ||||
| Keyword.Type: 'kt', | ||||
| Name: 'n', | ||||
| Name.Attribute: 'na', | ||||
| Name.Builtin: 'nb', | ||||
| Name.Builtin.Pseudo: 'bp', | ||||
| Name.Class: 'nc', | ||||
| Name.Constant: 'no', | ||||
| Name.Decorator: 'nd', | ||||
| Name.Entity: 'ni', | ||||
| Name.Exception: 'ne', | ||||
| Name.Function: 'nf', | ||||
| Name.Property: 'py', | ||||
| Name.Label: 'nl', | ||||
| Name.Namespace: 'nn', | ||||
| Name.Other: 'nx', | ||||
| Name.Tag: 'nt', | ||||
| Name.Variable: 'nv', | ||||
| Name.Variable.Class: 'vc', | ||||
| Name.Variable.Global: 'vg', | ||||
| Name.Variable.Instance: 'vi', | ||||
| Literal: 'l', | ||||
| Literal.Date: 'ld', | ||||
| String: 's', | ||||
| String.Backtick: 'sb', | ||||
| String.Char: 'sc', | ||||
| String.Doc: 'sd', | ||||
| String.Double: 's2', | ||||
| String.Escape: 'se', | ||||
| String.Heredoc: 'sh', | ||||
| String.Interpol: 'si', | ||||
| String.Other: 'sx', | ||||
| String.Regex: 'sr', | ||||
| String.Single: 's1', | ||||
| String.Symbol: 'ss', | ||||
| Number: 'm', | ||||
| Number.Float: 'mf', | ||||
| Number.Hex: 'mh', | ||||
| Number.Integer: 'mi', | ||||
| Number.Integer.Long: 'il', | ||||
| Number.Oct: 'mo', | ||||
| Operator: 'o', | ||||
| Operator.Word: 'ow', | ||||
| Punctuation: 'p', | ||||
| Comment: 'c', | ||||
| Comment.Multiline: 'cm', | ||||
| Comment.Preproc: 'cp', | ||||
| Comment.Single: 'c1', | ||||
| Comment.Special: 'cs', | ||||
| Generic: 'g', | ||||
| Generic.Deleted: 'gd', | ||||
| Generic.Emph: 'ge', | ||||
| Generic.Error: 'gr', | ||||
| Generic.Heading: 'gh', | ||||
| Generic.Inserted: 'gi', | ||||
| Generic.Output: 'go', | ||||
| Generic.Prompt: 'gp', | ||||
| Generic.Strong: 'gs', | ||||
| Generic.Subheading: 'gu', | ||||
| Generic.Traceback: 'gt', | ||||
| } | ||||