/Lib/lib2to3/pgen2/token.py

http://unladen-swallow.googlecode.com/ · Python · 82 lines · 69 code · 8 blank · 5 comment · 2 complexity · 8fd1f5c3fc2ad1b2afa7e17064b0ba04 MD5 · raw file 

    

  1. #! /usr/bin/env python
    2. 

  2. 

  3. """Token constants (from "token.h")."""
    4. 

  4. 

  5. #  Taken from Python (r53757) and modified to include some tokens
    6. 

  6. #   originally monkeypatched in by pgen2.tokenize
    7. 

  7. 

  8. #--start constants--
    9. 

  9. ENDMARKER = 0
    10. 

  10. NAME = 1
    11. 

  11. NUMBER = 2
    12. 

  12. STRING = 3
    13. 

  13. NEWLINE = 4
    14. 

  14. INDENT = 5
    15. 

  15. DEDENT = 6
    16. 

  16. LPAR = 7
    17. 

  17. RPAR = 8
    18. 

  18. LSQB = 9
    19. 

  19. RSQB = 10
    20. 

  20. COLON = 11
    21. 

  21. COMMA = 12
    22. 

  22. SEMI = 13
    23. 

  23. PLUS = 14
    24. 

  24. MINUS = 15
    25. 

  25. STAR = 16
    26. 

  26. SLASH = 17
    27. 

  27. VBAR = 18
    28. 

  28. AMPER = 19
    29. 

  29. LESS = 20
    30. 

  30. GREATER = 21
    31. 

  31. EQUAL = 22
    32. 

  32. DOT = 23
    33. 

  33. PERCENT = 24
    34. 

  34. BACKQUOTE = 25
    35. 

  35. LBRACE = 26
    36. 

  36. RBRACE = 27
    37. 

  37. EQEQUAL = 28
    38. 

  38. NOTEQUAL = 29
    39. 

  39. LESSEQUAL = 30
    40. 

  40. GREATEREQUAL = 31
    41. 

  41. TILDE = 32
    42. 

  42. CIRCUMFLEX = 33
    43. 

  43. LEFTSHIFT = 34
    44. 

  44. RIGHTSHIFT = 35
    45. 

  45. DOUBLESTAR = 36
    46. 

  46. PLUSEQUAL = 37
    47. 

  47. MINEQUAL = 38
    48. 

  48. STAREQUAL = 39
    49. 

  49. SLASHEQUAL = 40
    50. 

  50. PERCENTEQUAL = 41
    51. 

  51. AMPEREQUAL = 42
    52. 

  52. VBAREQUAL = 43
    53. 

  53. CIRCUMFLEXEQUAL = 44
    54. 

  54. LEFTSHIFTEQUAL = 45
    55. 

  55. RIGHTSHIFTEQUAL = 46
    56. 

  56. DOUBLESTAREQUAL = 47
    57. 

  57. DOUBLESLASH = 48
    58. 

  58. DOUBLESLASHEQUAL = 49
    59. 

  59. AT = 50
    60. 

  60. OP = 51
    61. 

  61. COMMENT = 52
    62. 

  62. NL = 53
    63. 

  63. RARROW = 54
    64. 

  64. ERRORTOKEN = 55
    65. 

  65. N_TOKENS = 56
    66. 

  66. NT_OFFSET = 256
    67. 

  67. #--end constants--
    68. 

  68. 

  69. tok_name = {}
    70. 

  70. for _name, _value in globals().items():
    71. 

  71.     if type(_value) is type(0):
    72. 

  72.         tok_name[_value] = _name
    73. 

  73. 

  74. 

  75. def ISTERMINAL(x):
    76. 

  76.     return x < NT_OFFSET
    77. 

  77. 

  78. def ISNONTERMINAL(x):
    79. 

  79.     return x >= NT_OFFSET
    80. 

  80. 

  81. def ISEOF(x):
    82. 

  82.     return x == ENDMARKER
    83.