/Lib/lib2to3/pgen2/grammar.py

http://unladen-swallow.googlecode.com/ · Python · 171 lines · 158 code · 8 blank · 5 comment · 8 complexity · 612ee8e1a84660a7c44f7d5af3e7db69 MD5 · raw file

  1. # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
  2. # Licensed to PSF under a Contributor Agreement.
  3. """This module defines the data structures used to represent a grammar.
  4. These are a bit arcane because they are derived from the data
  5. structures used by Python's 'pgen' parser generator.
  6. There's also a table here mapping operators to their names in the
  7. token module; the Python tokenize module reports all operators as the
  8. fallback token code OP, but the parser needs the actual token code.
  9. """
  10. # Python imports
  11. import pickle
  12. # Local imports
  13. from . import token, tokenize
  14. class Grammar(object):
  15. """Pgen parsing tables tables conversion class.
  16. Once initialized, this class supplies the grammar tables for the
  17. parsing engine implemented by parse.py. The parsing engine
  18. accesses the instance variables directly. The class here does not
  19. provide initialization of the tables; several subclasses exist to
  20. do this (see the conv and pgen modules).
  21. The load() method reads the tables from a pickle file, which is
  22. much faster than the other ways offered by subclasses. The pickle
  23. file is written by calling dump() (after loading the grammar
  24. tables using a subclass). The report() method prints a readable
  25. representation of the tables to stdout, for debugging.
  26. The instance variables are as follows:
  27. symbol2number -- a dict mapping symbol names to numbers. Symbol
  28. numbers are always 256 or higher, to distinguish
  29. them from token numbers, which are between 0 and
  30. 255 (inclusive).
  31. number2symbol -- a dict mapping numbers to symbol names;
  32. these two are each other's inverse.
  33. states -- a list of DFAs, where each DFA is a list of
  34. states, each state is is a list of arcs, and each
  35. arc is a (i, j) pair where i is a label and j is
  36. a state number. The DFA number is the index into
  37. this list. (This name is slightly confusing.)
  38. Final states are represented by a special arc of
  39. the form (0, j) where j is its own state number.
  40. dfas -- a dict mapping symbol numbers to (DFA, first)
  41. pairs, where DFA is an item from the states list
  42. above, and first is a set of tokens that can
  43. begin this grammar rule (represented by a dict
  44. whose values are always 1).
  45. labels -- a list of (x, y) pairs where x is either a token
  46. number or a symbol number, and y is either None
  47. or a string; the strings are keywords. The label
  48. number is the index in this list; label numbers
  49. are used to mark state transitions (arcs) in the
  50. DFAs.
  51. start -- the number of the grammar's start symbol.
  52. keywords -- a dict mapping keyword strings to arc labels.
  53. tokens -- a dict mapping token numbers to arc labels.
  54. """
  55. def __init__(self):
  56. self.symbol2number = {}
  57. self.number2symbol = {}
  58. self.states = []
  59. self.dfas = {}
  60. self.labels = [(0, "EMPTY")]
  61. self.keywords = {}
  62. self.tokens = {}
  63. self.symbol2label = {}
  64. self.start = 256
  65. def dump(self, filename):
  66. """Dump the grammar tables to a pickle file."""
  67. f = open(filename, "wb")
  68. pickle.dump(self.__dict__, f, 2)
  69. f.close()
  70. def load(self, filename):
  71. """Load the grammar tables from a pickle file."""
  72. f = open(filename, "rb")
  73. d = pickle.load(f)
  74. f.close()
  75. self.__dict__.update(d)
  76. def report(self):
  77. """Dump the grammar tables to standard output, for debugging."""
  78. from pprint import pprint
  79. print "s2n"
  80. pprint(self.symbol2number)
  81. print "n2s"
  82. pprint(self.number2symbol)
  83. print "states"
  84. pprint(self.states)
  85. print "dfas"
  86. pprint(self.dfas)
  87. print "labels"
  88. pprint(self.labels)
  89. print "start", self.start
  90. # Map from operator to number (since tokenize doesn't do this)
  91. opmap_raw = """
  92. ( LPAR
  93. ) RPAR
  94. [ LSQB
  95. ] RSQB
  96. : COLON
  97. , COMMA
  98. ; SEMI
  99. + PLUS
  100. - MINUS
  101. * STAR
  102. / SLASH
  103. | VBAR
  104. & AMPER
  105. < LESS
  106. > GREATER
  107. = EQUAL
  108. . DOT
  109. % PERCENT
  110. ` BACKQUOTE
  111. { LBRACE
  112. } RBRACE
  113. @ AT
  114. == EQEQUAL
  115. != NOTEQUAL
  116. <> NOTEQUAL
  117. <= LESSEQUAL
  118. >= GREATEREQUAL
  119. ~ TILDE
  120. ^ CIRCUMFLEX
  121. << LEFTSHIFT
  122. >> RIGHTSHIFT
  123. ** DOUBLESTAR
  124. += PLUSEQUAL
  125. -= MINEQUAL
  126. *= STAREQUAL
  127. /= SLASHEQUAL
  128. %= PERCENTEQUAL
  129. &= AMPEREQUAL
  130. |= VBAREQUAL
  131. ^= CIRCUMFLEXEQUAL
  132. <<= LEFTSHIFTEQUAL
  133. >>= RIGHTSHIFTEQUAL
  134. **= DOUBLESTAREQUAL
  135. // DOUBLESLASH
  136. //= DOUBLESLASHEQUAL
  137. -> RARROW
  138. """
  139. opmap = {}
  140. for line in opmap_raw.splitlines():
  141. if line:
  142. op, name = line.split()
  143. opmap[op] = getattr(token, name)