/Lib/test/test_ast.py

  1import sys, itertools, unittest
  2from test import test_support
  3import ast
  4
  5def to_tuple(t):
  6    if t is None or isinstance(t, (basestring, int, long, complex)):
  7        return t
  8    elif isinstance(t, list):
  9        return [to_tuple(e) for e in t]
 10    result = [t.__class__.__name__]
 11    if hasattr(t, 'lineno') and hasattr(t, 'col_offset'):
 12        result.append((t.lineno, t.col_offset))
 13    if t._fields is None:
 14        return tuple(result)
 15    for f in t._fields:
 16        result.append(to_tuple(getattr(t, f)))
 17    return tuple(result)
 18
 19
 20# These tests are compiled through "exec"
 21# There should be atleast one test per statement
 22exec_tests = [
 23    # FunctionDef
 24    "def f(): pass",
 25    # ClassDef
 26    "class C:pass",
 27    # Return
 28    "def f():return 1",
 29    # Delete
 30    "del v",
 31    # Assign
 32    "v = 1",
 33    # AugAssign
 34    "v += 1",
 35    # Print
 36    "print >>f, 1, ",
 37    # For
 38    "for v in v:pass",
 39    # While
 40    "while v:pass",
 41    # If
 42    "if v:pass",
 43    # Raise
 44    "raise Exception, 'string'",
 45    # TryExcept
 46    "try:\n  pass\nexcept Exception:\n  pass",
 47    # TryFinally
 48    "try:\n  pass\nfinally:\n  pass",
 49    # Assert
 50    "assert v",
 51    # Import
 52    "import sys",
 53    # ImportFrom
 54    "from sys import v",
 55    # Exec
 56    "exec 'v'",
 57    # Global
 58    "global v",
 59    # Expr
 60    "1",
 61    # Pass,
 62    "pass",
 63    # Break
 64    "break",
 65    # Continue
 66    "continue",
 67]
 68
 69# These are compiled through "single"
 70# because of overlap with "eval", it just tests what
 71# can't be tested with "eval"
 72single_tests = [
 73    "1+2"
 74]
 75
 76# These are compiled through "eval"
 77# It should test all expressions
 78eval_tests = [
 79  # BoolOp
 80  "a and b",
 81  # BinOp
 82  "a + b",
 83  # UnaryOp
 84  "not v",
 85  # Lambda
 86  "lambda:None",
 87  # Dict
 88  "{ 1:2 }",
 89  # ListComp
 90  "[a for b in c if d]",
 91  # GeneratorExp
 92  "(a for b in c if d)",
 93  # Yield - yield expressions can't work outside a function
 94  #
 95  # Compare
 96  "1 < 2 < 3",
 97  # Call
 98  "f(1,2,c=3,*d,**e)",
 99  # Repr
100  "`v`",
101  # Num
102  "10L",
103  # Str
104  "'string'",
105  # Attribute
106  "a.b",
107  # Subscript
108  "a[b:c]",
109  # Name
110  "v",
111  # List
112  "[1,2,3]",
113  # Tuple
114  "1,2,3",
115  # Combination
116  "a.b.c.d(a.b[1:2])",
117
118]
119
120# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
121# excepthandler, arguments, keywords, alias
122
123class AST_Tests(unittest.TestCase):
124
125    def _assert_order(self, ast_node, parent_pos):
126        if not isinstance(ast_node, ast.AST) or ast_node._fields is None:
127            return
128        if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)):
129            node_pos = (ast_node.lineno, ast_node.col_offset)
130            self.assert_(node_pos >= parent_pos)
131            parent_pos = (ast_node.lineno, ast_node.col_offset)
132        for name in ast_node._fields:
133            value = getattr(ast_node, name)
134            if isinstance(value, list):
135                for child in value:
136                    self._assert_order(child, parent_pos)
137            elif value is not None:
138                self._assert_order(value, parent_pos)
139
140    def test_snippets(self):
141        for input, output, kind in ((exec_tests, exec_results, "exec"),
142                                    (single_tests, single_results, "single"),
143                                    (eval_tests, eval_results, "eval")):
144            for i, o in itertools.izip(input, output):
145                ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST)
146                self.assertEquals(to_tuple(ast_tree), o)
147                self._assert_order(ast_tree, (0, 0))
148
149    def test_nodeclasses(self):
150        x = ast.BinOp(1, 2, 3, lineno=0)
151        self.assertEquals(x.left, 1)
152        self.assertEquals(x.op, 2)
153        self.assertEquals(x.right, 3)
154        self.assertEquals(x.lineno, 0)
155
156        # node raises exception when not given enough arguments
157        self.assertRaises(TypeError, ast.BinOp, 1, 2)
158
159        # can set attributes through kwargs too
160        x = ast.BinOp(left=1, op=2, right=3, lineno=0)
161        self.assertEquals(x.left, 1)
162        self.assertEquals(x.op, 2)
163        self.assertEquals(x.right, 3)
164        self.assertEquals(x.lineno, 0)
165
166        # this used to fail because Sub._fields was None
167        x = ast.Sub()
168
169    def test_pickling(self):
170        import pickle
171        mods = [pickle]
172        try:
173            import cPickle
174            mods.append(cPickle)
175        except ImportError:
176            pass
177        protocols = [0, 1, 2]
178        for mod in mods:
179            for protocol in protocols:
180                for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests):
181                    ast2 = mod.loads(mod.dumps(ast, protocol))
182                    self.assertEquals(to_tuple(ast2), to_tuple(ast))
183
184
185class ASTHelpers_Test(unittest.TestCase):
186
187    def test_parse(self):
188        a = ast.parse('foo(1 + 1)')
189        b = compile('foo(1 + 1)', '<unknown>', 'exec', ast.PyCF_ONLY_AST)
190        self.assertEqual(ast.dump(a), ast.dump(b))
191
192    def test_dump(self):
193        node = ast.parse('spam(eggs, "and cheese")')
194        self.assertEqual(ast.dump(node),
195            "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
196            "args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
197            "keywords=[], starargs=None, kwargs=None))])"
198        )
199        self.assertEqual(ast.dump(node, annotate_fields=False),
200            "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
201            "Str('and cheese')], [], None, None))])"
202        )
203        self.assertEqual(ast.dump(node, include_attributes=True),
204            "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
205            "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
206            "lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
207            "col_offset=11)], keywords=[], starargs=None, kwargs=None, "
208            "lineno=1, col_offset=0), lineno=1, col_offset=0)])"
209        )
210
211    def test_copy_location(self):
212        src = ast.parse('1 + 1', mode='eval')
213        src.body.right = ast.copy_location(ast.Num(2), src.body.right)
214        self.assertEqual(ast.dump(src, include_attributes=True),
215            'Expression(body=BinOp(left=Num(n=1, lineno=1, col_offset=0), '
216            'op=Add(), right=Num(n=2, lineno=1, col_offset=4), lineno=1, '
217            'col_offset=0))'
218        )
219
220    def test_fix_missing_locations(self):
221        src = ast.parse('write("spam")')
222        src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()),
223                                          [ast.Str('eggs')], [], None, None)))
224        self.assertEqual(src, ast.fix_missing_locations(src))
225        self.assertEqual(ast.dump(src, include_attributes=True),
226            "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
227            "lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, "
228            "col_offset=6)], keywords=[], starargs=None, kwargs=None, "
229            "lineno=1, col_offset=0), lineno=1, col_offset=0), "
230            "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
231            "col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], "
232            "keywords=[], starargs=None, kwargs=None, lineno=1, "
233            "col_offset=0), lineno=1, col_offset=0)])"
234        )
235
236    def test_increment_lineno(self):
237        src = ast.parse('1 + 1', mode='eval')
238        self.assertEqual(ast.increment_lineno(src, n=3), src)
239        self.assertEqual(ast.dump(src, include_attributes=True),
240            'Expression(body=BinOp(left=Num(n=1, lineno=4, col_offset=0), '
241            'op=Add(), right=Num(n=1, lineno=4, col_offset=4), lineno=4, '
242            'col_offset=0))'
243        )
244
245    def test_iter_fields(self):
246        node = ast.parse('foo()', mode='eval')
247        d = dict(ast.iter_fields(node.body))
248        self.assertEqual(d.pop('func').id, 'foo')
249        self.assertEqual(d, {'keywords': [], 'kwargs': None,
250                             'args': [], 'starargs': None})
251
252    def test_iter_child_nodes(self):
253        node = ast.parse("spam(23, 42, eggs='leek')", mode='eval')
254        self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4)
255        iterator = ast.iter_child_nodes(node.body)
256        self.assertEqual(next(iterator).id, 'spam')
257        self.assertEqual(next(iterator).n, 23)
258        self.assertEqual(next(iterator).n, 42)
259        self.assertEqual(ast.dump(next(iterator)),
260            "keyword(arg='eggs', value=Str(s='leek'))"
261        )
262
263    def test_get_docstring(self):
264        node = ast.parse('def foo():\n  """line one\n  line two"""')
265        self.assertEqual(ast.get_docstring(node.body[0]),
266                         'line one\nline two')
267
268    def test_literal_eval(self):
269        self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3])
270        self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42})
271        self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None))
272        self.assertRaises(ValueError, ast.literal_eval, 'foo()')
273
274def test_main():
275    test_support.run_unittest(AST_Tests, ASTHelpers_Test)
276
277def main():
278    if __name__ != '__main__':
279        return
280    if sys.argv[1:] == ['-g']:
281        for statements, kind in ((exec_tests, "exec"), (single_tests, "single"),
282                                 (eval_tests, "eval")):
283            print kind+"_results = ["
284            for s in statements:
285                print repr(to_tuple(compile(s, "?", kind, 0x400)))+","
286                print "]"
287        print "main()"
288        raise SystemExit
289    test_main()
290
291#### EVERYTHING BELOW IS GENERATED #####
292exec_results = [
293('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
294('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))], [])]),
295('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
296('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
297('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
298('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
299('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]),
300('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
301('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
302('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
303('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]),
304('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
305('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
306('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
307('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
308('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
309('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]),
310('Module', [('Global', (1, 0), ['v'])]),
311('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
312('Module', [('Pass', (1, 0))]),
313('Module', [('Break', (1, 0))]),
314('Module', [('Continue', (1, 0))]),
315]
316single_results = [
317('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
318]
319eval_results = [
320('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
321('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
322('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
323('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
324('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
325('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
326('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
327('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
328('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
329('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))),
330('Expression', ('Num', (1, 0), 10L)),
331('Expression', ('Str', (1, 0), 'string')),
332('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
333('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
334('Expression', ('Name', (1, 0), 'v', ('Load',))),
335('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
336('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
337('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
338]
339main()