PageRenderTime 21ms CodeModel.GetById 11ms app.highlight 7ms RepoModel.GetById 1ms app.codeStats 0ms

/Lib/lib2to3/pgen2/token.py

http://unladen-swallow.googlecode.com/
Python | 82 lines | 80 code | 1 blank | 1 comment | 0 complexity | 8fd1f5c3fc2ad1b2afa7e17064b0ba04 MD5 | raw file
 1#! /usr/bin/env python
 2
 3"""Token constants (from "token.h")."""
 4
 5#  Taken from Python (r53757) and modified to include some tokens
 6#   originally monkeypatched in by pgen2.tokenize
 7
 8#--start constants--
 9ENDMARKER = 0
10NAME = 1
11NUMBER = 2
12STRING = 3
13NEWLINE = 4
14INDENT = 5
15DEDENT = 6
16LPAR = 7
17RPAR = 8
18LSQB = 9
19RSQB = 10
20COLON = 11
21COMMA = 12
22SEMI = 13
23PLUS = 14
24MINUS = 15
25STAR = 16
26SLASH = 17
27VBAR = 18
28AMPER = 19
29LESS = 20
30GREATER = 21
31EQUAL = 22
32DOT = 23
33PERCENT = 24
34BACKQUOTE = 25
35LBRACE = 26
36RBRACE = 27
37EQEQUAL = 28
38NOTEQUAL = 29
39LESSEQUAL = 30
40GREATEREQUAL = 31
41TILDE = 32
42CIRCUMFLEX = 33
43LEFTSHIFT = 34
44RIGHTSHIFT = 35
45DOUBLESTAR = 36
46PLUSEQUAL = 37
47MINEQUAL = 38
48STAREQUAL = 39
49SLASHEQUAL = 40
50PERCENTEQUAL = 41
51AMPEREQUAL = 42
52VBAREQUAL = 43
53CIRCUMFLEXEQUAL = 44
54LEFTSHIFTEQUAL = 45
55RIGHTSHIFTEQUAL = 46
56DOUBLESTAREQUAL = 47
57DOUBLESLASH = 48
58DOUBLESLASHEQUAL = 49
59AT = 50
60OP = 51
61COMMENT = 52
62NL = 53
63RARROW = 54
64ERRORTOKEN = 55
65N_TOKENS = 56
66NT_OFFSET = 256
67#--end constants--
68
69tok_name = {}
70for _name, _value in globals().items():
71    if type(_value) is type(0):
72        tok_name[_value] = _name
73
74
75def ISTERMINAL(x):
76    return x < NT_OFFSET
77
78def ISNONTERMINAL(x):
79    return x >= NT_OFFSET
80
81def ISEOF(x):
82    return x == ENDMARKER