/Lib/test/test_ast.py

http://unladen-swallow.googlecode.com/ · Python · 339 lines · 255 code · 29 blank · 55 comment · 27 complexity · 3ec084dbfb8c45646f152c952a412b21 MD5 · raw file

  1. import sys, itertools, unittest
  2. from test import test_support
  3. import ast
  4. def to_tuple(t):
  5. if t is None or isinstance(t, (basestring, int, long, complex)):
  6. return t
  7. elif isinstance(t, list):
  8. return [to_tuple(e) for e in t]
  9. result = [t.__class__.__name__]
  10. if hasattr(t, 'lineno') and hasattr(t, 'col_offset'):
  11. result.append((t.lineno, t.col_offset))
  12. if t._fields is None:
  13. return tuple(result)
  14. for f in t._fields:
  15. result.append(to_tuple(getattr(t, f)))
  16. return tuple(result)
  17. # These tests are compiled through "exec"
  18. # There should be atleast one test per statement
  19. exec_tests = [
  20. # FunctionDef
  21. "def f(): pass",
  22. # ClassDef
  23. "class C:pass",
  24. # Return
  25. "def f():return 1",
  26. # Delete
  27. "del v",
  28. # Assign
  29. "v = 1",
  30. # AugAssign
  31. "v += 1",
  32. # Print
  33. "print >>f, 1, ",
  34. # For
  35. "for v in v:pass",
  36. # While
  37. "while v:pass",
  38. # If
  39. "if v:pass",
  40. # Raise
  41. "raise Exception, 'string'",
  42. # TryExcept
  43. "try:\n pass\nexcept Exception:\n pass",
  44. # TryFinally
  45. "try:\n pass\nfinally:\n pass",
  46. # Assert
  47. "assert v",
  48. # Import
  49. "import sys",
  50. # ImportFrom
  51. "from sys import v",
  52. # Exec
  53. "exec 'v'",
  54. # Global
  55. "global v",
  56. # Expr
  57. "1",
  58. # Pass,
  59. "pass",
  60. # Break
  61. "break",
  62. # Continue
  63. "continue",
  64. ]
  65. # These are compiled through "single"
  66. # because of overlap with "eval", it just tests what
  67. # can't be tested with "eval"
  68. single_tests = [
  69. "1+2"
  70. ]
  71. # These are compiled through "eval"
  72. # It should test all expressions
  73. eval_tests = [
  74. # BoolOp
  75. "a and b",
  76. # BinOp
  77. "a + b",
  78. # UnaryOp
  79. "not v",
  80. # Lambda
  81. "lambda:None",
  82. # Dict
  83. "{ 1:2 }",
  84. # ListComp
  85. "[a for b in c if d]",
  86. # GeneratorExp
  87. "(a for b in c if d)",
  88. # Yield - yield expressions can't work outside a function
  89. #
  90. # Compare
  91. "1 < 2 < 3",
  92. # Call
  93. "f(1,2,c=3,*d,**e)",
  94. # Repr
  95. "`v`",
  96. # Num
  97. "10L",
  98. # Str
  99. "'string'",
  100. # Attribute
  101. "a.b",
  102. # Subscript
  103. "a[b:c]",
  104. # Name
  105. "v",
  106. # List
  107. "[1,2,3]",
  108. # Tuple
  109. "1,2,3",
  110. # Combination
  111. "a.b.c.d(a.b[1:2])",
  112. ]
  113. # TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
  114. # excepthandler, arguments, keywords, alias
  115. class AST_Tests(unittest.TestCase):
  116. def _assert_order(self, ast_node, parent_pos):
  117. if not isinstance(ast_node, ast.AST) or ast_node._fields is None:
  118. return
  119. if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)):
  120. node_pos = (ast_node.lineno, ast_node.col_offset)
  121. self.assert_(node_pos >= parent_pos)
  122. parent_pos = (ast_node.lineno, ast_node.col_offset)
  123. for name in ast_node._fields:
  124. value = getattr(ast_node, name)
  125. if isinstance(value, list):
  126. for child in value:
  127. self._assert_order(child, parent_pos)
  128. elif value is not None:
  129. self._assert_order(value, parent_pos)
  130. def test_snippets(self):
  131. for input, output, kind in ((exec_tests, exec_results, "exec"),
  132. (single_tests, single_results, "single"),
  133. (eval_tests, eval_results, "eval")):
  134. for i, o in itertools.izip(input, output):
  135. ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST)
  136. self.assertEquals(to_tuple(ast_tree), o)
  137. self._assert_order(ast_tree, (0, 0))
  138. def test_nodeclasses(self):
  139. x = ast.BinOp(1, 2, 3, lineno=0)
  140. self.assertEquals(x.left, 1)
  141. self.assertEquals(x.op, 2)
  142. self.assertEquals(x.right, 3)
  143. self.assertEquals(x.lineno, 0)
  144. # node raises exception when not given enough arguments
  145. self.assertRaises(TypeError, ast.BinOp, 1, 2)
  146. # can set attributes through kwargs too
  147. x = ast.BinOp(left=1, op=2, right=3, lineno=0)
  148. self.assertEquals(x.left, 1)
  149. self.assertEquals(x.op, 2)
  150. self.assertEquals(x.right, 3)
  151. self.assertEquals(x.lineno, 0)
  152. # this used to fail because Sub._fields was None
  153. x = ast.Sub()
  154. def test_pickling(self):
  155. import pickle
  156. mods = [pickle]
  157. try:
  158. import cPickle
  159. mods.append(cPickle)
  160. except ImportError:
  161. pass
  162. protocols = [0, 1, 2]
  163. for mod in mods:
  164. for protocol in protocols:
  165. for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests):
  166. ast2 = mod.loads(mod.dumps(ast, protocol))
  167. self.assertEquals(to_tuple(ast2), to_tuple(ast))
  168. class ASTHelpers_Test(unittest.TestCase):
  169. def test_parse(self):
  170. a = ast.parse('foo(1 + 1)')
  171. b = compile('foo(1 + 1)', '<unknown>', 'exec', ast.PyCF_ONLY_AST)
  172. self.assertEqual(ast.dump(a), ast.dump(b))
  173. def test_dump(self):
  174. node = ast.parse('spam(eggs, "and cheese")')
  175. self.assertEqual(ast.dump(node),
  176. "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
  177. "args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
  178. "keywords=[], starargs=None, kwargs=None))])"
  179. )
  180. self.assertEqual(ast.dump(node, annotate_fields=False),
  181. "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
  182. "Str('and cheese')], [], None, None))])"
  183. )
  184. self.assertEqual(ast.dump(node, include_attributes=True),
  185. "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
  186. "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
  187. "lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
  188. "col_offset=11)], keywords=[], starargs=None, kwargs=None, "
  189. "lineno=1, col_offset=0), lineno=1, col_offset=0)])"
  190. )
  191. def test_copy_location(self):
  192. src = ast.parse('1 + 1', mode='eval')
  193. src.body.right = ast.copy_location(ast.Num(2), src.body.right)
  194. self.assertEqual(ast.dump(src, include_attributes=True),
  195. 'Expression(body=BinOp(left=Num(n=1, lineno=1, col_offset=0), '
  196. 'op=Add(), right=Num(n=2, lineno=1, col_offset=4), lineno=1, '
  197. 'col_offset=0))'
  198. )
  199. def test_fix_missing_locations(self):
  200. src = ast.parse('write("spam")')
  201. src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()),
  202. [ast.Str('eggs')], [], None, None)))
  203. self.assertEqual(src, ast.fix_missing_locations(src))
  204. self.assertEqual(ast.dump(src, include_attributes=True),
  205. "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
  206. "lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, "
  207. "col_offset=6)], keywords=[], starargs=None, kwargs=None, "
  208. "lineno=1, col_offset=0), lineno=1, col_offset=0), "
  209. "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
  210. "col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], "
  211. "keywords=[], starargs=None, kwargs=None, lineno=1, "
  212. "col_offset=0), lineno=1, col_offset=0)])"
  213. )
  214. def test_increment_lineno(self):
  215. src = ast.parse('1 + 1', mode='eval')
  216. self.assertEqual(ast.increment_lineno(src, n=3), src)
  217. self.assertEqual(ast.dump(src, include_attributes=True),
  218. 'Expression(body=BinOp(left=Num(n=1, lineno=4, col_offset=0), '
  219. 'op=Add(), right=Num(n=1, lineno=4, col_offset=4), lineno=4, '
  220. 'col_offset=0))'
  221. )
  222. def test_iter_fields(self):
  223. node = ast.parse('foo()', mode='eval')
  224. d = dict(ast.iter_fields(node.body))
  225. self.assertEqual(d.pop('func').id, 'foo')
  226. self.assertEqual(d, {'keywords': [], 'kwargs': None,
  227. 'args': [], 'starargs': None})
  228. def test_iter_child_nodes(self):
  229. node = ast.parse("spam(23, 42, eggs='leek')", mode='eval')
  230. self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4)
  231. iterator = ast.iter_child_nodes(node.body)
  232. self.assertEqual(next(iterator).id, 'spam')
  233. self.assertEqual(next(iterator).n, 23)
  234. self.assertEqual(next(iterator).n, 42)
  235. self.assertEqual(ast.dump(next(iterator)),
  236. "keyword(arg='eggs', value=Str(s='leek'))"
  237. )
  238. def test_get_docstring(self):
  239. node = ast.parse('def foo():\n """line one\n line two"""')
  240. self.assertEqual(ast.get_docstring(node.body[0]),
  241. 'line one\nline two')
  242. def test_literal_eval(self):
  243. self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3])
  244. self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42})
  245. self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None))
  246. self.assertRaises(ValueError, ast.literal_eval, 'foo()')
  247. def test_main():
  248. test_support.run_unittest(AST_Tests, ASTHelpers_Test)
  249. def main():
  250. if __name__ != '__main__':
  251. return
  252. if sys.argv[1:] == ['-g']:
  253. for statements, kind in ((exec_tests, "exec"), (single_tests, "single"),
  254. (eval_tests, "eval")):
  255. print kind+"_results = ["
  256. for s in statements:
  257. print repr(to_tuple(compile(s, "?", kind, 0x400)))+","
  258. print "]"
  259. print "main()"
  260. raise SystemExit
  261. test_main()
  262. #### EVERYTHING BELOW IS GENERATED #####
  263. exec_results = [
  264. ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
  265. ('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))], [])]),
  266. ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
  267. ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
  268. ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
  269. ('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
  270. ('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]),
  271. ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
  272. ('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
  273. ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
  274. ('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]),
  275. ('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
  276. ('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
  277. ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
  278. ('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
  279. ('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
  280. ('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]),
  281. ('Module', [('Global', (1, 0), ['v'])]),
  282. ('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
  283. ('Module', [('Pass', (1, 0))]),
  284. ('Module', [('Break', (1, 0))]),
  285. ('Module', [('Continue', (1, 0))]),
  286. ]
  287. single_results = [
  288. ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
  289. ]
  290. eval_results = [
  291. ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
  292. ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
  293. ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
  294. ('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
  295. ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
  296. ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
  297. ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
  298. ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
  299. ('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
  300. ('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))),
  301. ('Expression', ('Num', (1, 0), 10L)),
  302. ('Expression', ('Str', (1, 0), 'string')),
  303. ('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
  304. ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
  305. ('Expression', ('Name', (1, 0), 'v', ('Load',))),
  306. ('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
  307. ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
  308. ('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
  309. ]
  310. main()