PageRenderTime 937ms CodeModel.GetById 121ms app.highlight 626ms RepoModel.GetById 138ms app.codeStats 1ms

/Lib/compiler/transformer.py

http://unladen-swallow.googlecode.com/
Python | 1498 lines | 1411 code | 28 blank | 59 comment | 35 complexity | 4f654a5b70300e8c17f100aa3633e68e MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1"""Parse tree transformation module.
   2
   3Transforms Python source code into an abstract syntax tree (AST)
   4defined in the ast module.
   5
   6The simplest ways to invoke this module are via parse and parseFile.
   7parse(buf) -> AST
   8parseFile(path) -> AST
   9"""
  10
  11# Original version written by Greg Stein (gstein@lyra.org)
  12#                         and Bill Tutt (rassilon@lima.mudlib.org)
  13# February 1997.
  14#
  15# Modifications and improvements for Python 2.0 by Jeremy Hylton and
  16# Mark Hammond
  17#
  18# Some fixes to try to have correct line number on almost all nodes
  19# (except Module, Discard and Stmt) added by Sylvain Thenault
  20#
  21# Portions of this file are:
  22# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
  23#
  24# This module is provided under a BSD-ish license. See
  25#   http://www.opensource.org/licenses/bsd-license.html
  26# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
  27
  28from compiler.ast import *
  29import parser
  30import symbol
  31import token
  32
  33class WalkerError(StandardError):
  34    pass
  35
  36from compiler.consts import CO_VARARGS, CO_VARKEYWORDS
  37from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY
  38
  39def parseFile(path):
  40    f = open(path, "U")
  41    # XXX The parser API tolerates files without a trailing newline,
  42    # but not strings without a trailing newline.  Always add an extra
  43    # newline to the file contents, since we're going through the string
  44    # version of the API.
  45    src = f.read() + "\n"
  46    f.close()
  47    return parse(src)
  48
  49def parse(buf, mode="exec"):
  50    if mode == "exec" or mode == "single":
  51        return Transformer().parsesuite(buf)
  52    elif mode == "eval":
  53        return Transformer().parseexpr(buf)
  54    else:
  55        raise ValueError("compile() arg 3 must be"
  56                         " 'exec' or 'eval' or 'single'")
  57
  58def asList(nodes):
  59    l = []
  60    for item in nodes:
  61        if hasattr(item, "asList"):
  62            l.append(item.asList())
  63        else:
  64            if type(item) is type( (None, None) ):
  65                l.append(tuple(asList(item)))
  66            elif type(item) is type( [] ):
  67                l.append(asList(item))
  68            else:
  69                l.append(item)
  70    return l
  71
  72def extractLineNo(ast):
  73    if not isinstance(ast[1], tuple):
  74        # get a terminal node
  75        return ast[2]
  76    for child in ast[1:]:
  77        if isinstance(child, tuple):
  78            lineno = extractLineNo(child)
  79            if lineno is not None:
  80                return lineno
  81
  82def Node(*args):
  83    kind = args[0]
  84    if kind in nodes:
  85        try:
  86            return nodes[kind](*args[1:])
  87        except TypeError:
  88            print nodes[kind], len(args), args
  89            raise
  90    else:
  91        raise WalkerError, "Can't find appropriate Node type: %s" % str(args)
  92        #return apply(ast.Node, args)
  93
  94class Transformer:
  95    """Utility object for transforming Python parse trees.
  96
  97    Exposes the following methods:
  98        tree = transform(ast_tree)
  99        tree = parsesuite(text)
 100        tree = parseexpr(text)
 101        tree = parsefile(fileob | filename)
 102    """
 103
 104    def __init__(self):
 105        self._dispatch = {}
 106        for value, name in symbol.sym_name.items():
 107            if hasattr(self, name):
 108                self._dispatch[value] = getattr(self, name)
 109        self._dispatch[token.NEWLINE] = self.com_NEWLINE
 110        self._atom_dispatch = {token.LPAR: self.atom_lpar,
 111                               token.LSQB: self.atom_lsqb,
 112                               token.LBRACE: self.atom_lbrace,
 113                               token.BACKQUOTE: self.atom_backquote,
 114                               token.NUMBER: self.atom_number,
 115                               token.STRING: self.atom_string,
 116                               token.NAME: self.atom_name,
 117                               }
 118        self.encoding = None
 119
 120    def transform(self, tree):
 121        """Transform an AST into a modified parse tree."""
 122        if not (isinstance(tree, tuple) or isinstance(tree, list)):
 123            tree = parser.st2tuple(tree, line_info=1)
 124        return self.compile_node(tree)
 125
 126    def parsesuite(self, text):
 127        """Return a modified parse tree for the given suite text."""
 128        return self.transform(parser.suite(text))
 129
 130    def parseexpr(self, text):
 131        """Return a modified parse tree for the given expression text."""
 132        return self.transform(parser.expr(text))
 133
 134    def parsefile(self, file):
 135        """Return a modified parse tree for the contents of the given file."""
 136        if type(file) == type(''):
 137            file = open(file)
 138        return self.parsesuite(file.read())
 139
 140    # --------------------------------------------------------------
 141    #
 142    # PRIVATE METHODS
 143    #
 144
 145    def compile_node(self, node):
 146        ### emit a line-number node?
 147        n = node[0]
 148
 149        if n == symbol.encoding_decl:
 150            self.encoding = node[2]
 151            node = node[1]
 152            n = node[0]
 153
 154        if n == symbol.single_input:
 155            return self.single_input(node[1:])
 156        if n == symbol.file_input:
 157            return self.file_input(node[1:])
 158        if n == symbol.eval_input:
 159            return self.eval_input(node[1:])
 160        if n == symbol.lambdef:
 161            return self.lambdef(node[1:])
 162        if n == symbol.funcdef:
 163            return self.funcdef(node[1:])
 164        if n == symbol.classdef:
 165            return self.classdef(node[1:])
 166
 167        raise WalkerError, ('unexpected node type', n)
 168
 169    def single_input(self, node):
 170        ### do we want to do anything about being "interactive" ?
 171
 172        # NEWLINE | simple_stmt | compound_stmt NEWLINE
 173        n = node[0][0]
 174        if n != token.NEWLINE:
 175            return self.com_stmt(node[0])
 176
 177        return Pass()
 178
 179    def file_input(self, nodelist):
 180        doc = self.get_docstring(nodelist, symbol.file_input)
 181        if doc is not None:
 182            i = 1
 183        else:
 184            i = 0
 185        stmts = []
 186        for node in nodelist[i:]:
 187            if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
 188                self.com_append_stmt(stmts, node)
 189        return Module(doc, Stmt(stmts))
 190
 191    def eval_input(self, nodelist):
 192        # from the built-in function input()
 193        ### is this sufficient?
 194        return Expression(self.com_node(nodelist[0]))
 195
 196    def decorator_name(self, nodelist):
 197        listlen = len(nodelist)
 198        assert listlen >= 1 and listlen % 2 == 1
 199
 200        item = self.atom_name(nodelist)
 201        i = 1
 202        while i < listlen:
 203            assert nodelist[i][0] == token.DOT
 204            assert nodelist[i + 1][0] == token.NAME
 205            item = Getattr(item, nodelist[i + 1][1])
 206            i += 2
 207
 208        return item
 209
 210    def decorator(self, nodelist):
 211        # '@' dotted_name [ '(' [arglist] ')' ]
 212        assert len(nodelist) in (3, 5, 6)
 213        assert nodelist[0][0] == token.AT
 214        assert nodelist[-1][0] == token.NEWLINE
 215
 216        assert nodelist[1][0] == symbol.dotted_name
 217        funcname = self.decorator_name(nodelist[1][1:])
 218
 219        if len(nodelist) > 3:
 220            assert nodelist[2][0] == token.LPAR
 221            expr = self.com_call_function(funcname, nodelist[3])
 222        else:
 223            expr = funcname
 224
 225        return expr
 226
 227    def decorators(self, nodelist):
 228        # decorators: decorator ([NEWLINE] decorator)* NEWLINE
 229        items = []
 230        for dec_nodelist in nodelist:
 231            assert dec_nodelist[0] == symbol.decorator
 232            items.append(self.decorator(dec_nodelist[1:]))
 233        return Decorators(items)
 234
 235    def decorated(self, nodelist):
 236        assert nodelist[0][0] == symbol.decorators
 237        if nodelist[1][0] == symbol.funcdef:
 238            n = [nodelist[0]] + list(nodelist[1][1:])
 239            return self.funcdef(n)
 240        elif nodelist[1][0] == symbol.classdef:
 241            decorators = self.decorators(nodelist[0][1:])
 242            cls = self.classdef(nodelist[1][1:])
 243            cls.decorators = decorators
 244            return cls
 245        raise WalkerError()
 246
 247    def funcdef(self, nodelist):
 248        #                    -6   -5    -4         -3  -2    -1
 249        # funcdef: [decorators] 'def' NAME parameters ':' suite
 250        # parameters: '(' [varargslist] ')'
 251
 252        if len(nodelist) == 6:
 253            assert nodelist[0][0] == symbol.decorators
 254            decorators = self.decorators(nodelist[0][1:])
 255        else:
 256            assert len(nodelist) == 5
 257            decorators = None
 258
 259        lineno = nodelist[-4][2]
 260        name = nodelist[-4][1]
 261        args = nodelist[-3][2]
 262
 263        if args[0] == symbol.varargslist:
 264            names, defaults, flags = self.com_arglist(args[1:])
 265        else:
 266            names = defaults = ()
 267            flags = 0
 268        doc = self.get_docstring(nodelist[-1])
 269
 270        # code for function
 271        code = self.com_node(nodelist[-1])
 272
 273        if doc is not None:
 274            assert isinstance(code, Stmt)
 275            assert isinstance(code.nodes[0], Discard)
 276            del code.nodes[0]
 277        return Function(decorators, name, names, defaults, flags, doc, code,
 278                     lineno=lineno)
 279
 280    def lambdef(self, nodelist):
 281        # lambdef: 'lambda' [varargslist] ':' test
 282        if nodelist[2][0] == symbol.varargslist:
 283            names, defaults, flags = self.com_arglist(nodelist[2][1:])
 284        else:
 285            names = defaults = ()
 286            flags = 0
 287
 288        # code for lambda
 289        code = self.com_node(nodelist[-1])
 290
 291        return Lambda(names, defaults, flags, code, lineno=nodelist[1][2])
 292    old_lambdef = lambdef
 293
 294    def classdef(self, nodelist):
 295        # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
 296
 297        name = nodelist[1][1]
 298        doc = self.get_docstring(nodelist[-1])
 299        if nodelist[2][0] == token.COLON:
 300            bases = []
 301        elif nodelist[3][0] == token.RPAR:
 302            bases = []
 303        else:
 304            bases = self.com_bases(nodelist[3])
 305
 306        # code for class
 307        code = self.com_node(nodelist[-1])
 308
 309        if doc is not None:
 310            assert isinstance(code, Stmt)
 311            assert isinstance(code.nodes[0], Discard)
 312            del code.nodes[0]
 313
 314        return Class(name, bases, doc, code, lineno=nodelist[1][2])
 315
 316    def stmt(self, nodelist):
 317        return self.com_stmt(nodelist[0])
 318
 319    small_stmt = stmt
 320    flow_stmt = stmt
 321    compound_stmt = stmt
 322
 323    def simple_stmt(self, nodelist):
 324        # small_stmt (';' small_stmt)* [';'] NEWLINE
 325        stmts = []
 326        for i in range(0, len(nodelist), 2):
 327            self.com_append_stmt(stmts, nodelist[i])
 328        return Stmt(stmts)
 329
 330    def parameters(self, nodelist):
 331        raise WalkerError
 332
 333    def varargslist(self, nodelist):
 334        raise WalkerError
 335
 336    def fpdef(self, nodelist):
 337        raise WalkerError
 338
 339    def fplist(self, nodelist):
 340        raise WalkerError
 341
 342    def dotted_name(self, nodelist):
 343        raise WalkerError
 344
 345    def comp_op(self, nodelist):
 346        raise WalkerError
 347
 348    def trailer(self, nodelist):
 349        raise WalkerError
 350
 351    def sliceop(self, nodelist):
 352        raise WalkerError
 353
 354    def argument(self, nodelist):
 355        raise WalkerError
 356
 357    # --------------------------------------------------------------
 358    #
 359    # STATEMENT NODES  (invoked by com_node())
 360    #
 361
 362    def expr_stmt(self, nodelist):
 363        # augassign testlist | testlist ('=' testlist)*
 364        en = nodelist[-1]
 365        exprNode = self.lookup_node(en)(en[1:])
 366        if len(nodelist) == 1:
 367            return Discard(exprNode, lineno=exprNode.lineno)
 368        if nodelist[1][0] == token.EQUAL:
 369            nodesl = []
 370            for i in range(0, len(nodelist) - 2, 2):
 371                nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
 372            return Assign(nodesl, exprNode, lineno=nodelist[1][2])
 373        else:
 374            lval = self.com_augassign(nodelist[0])
 375            op = self.com_augassign_op(nodelist[1])
 376            return AugAssign(lval, op[1], exprNode, lineno=op[2])
 377        raise WalkerError, "can't get here"
 378
 379    def print_stmt(self, nodelist):
 380        # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
 381        items = []
 382        if len(nodelist) == 1:
 383            start = 1
 384            dest = None
 385        elif nodelist[1][0] == token.RIGHTSHIFT:
 386            assert len(nodelist) == 3 \
 387                   or nodelist[3][0] == token.COMMA
 388            dest = self.com_node(nodelist[2])
 389            start = 4
 390        else:
 391            dest = None
 392            start = 1
 393        for i in range(start, len(nodelist), 2):
 394            items.append(self.com_node(nodelist[i]))
 395        newline = (nodelist[-1][0] != token.COMMA)
 396        return Print(items, dest, newline, lineno=nodelist[0][2])
 397
 398    def del_stmt(self, nodelist):
 399        return self.com_assign(nodelist[1], OP_DELETE)
 400
 401    def pass_stmt(self, nodelist):
 402        return Pass(lineno=nodelist[0][2])
 403
 404    def break_stmt(self, nodelist):
 405        return Break(lineno=nodelist[0][2])
 406
 407    def continue_stmt(self, nodelist):
 408        return Continue(lineno=nodelist[0][2])
 409
 410    def return_stmt(self, nodelist):
 411        # return: [testlist]
 412        if len(nodelist) < 2:
 413            return Return(Const(None), lineno=nodelist[0][2])
 414        return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
 415
 416    def yield_stmt(self, nodelist):
 417        expr = self.com_node(nodelist[0])
 418        return Discard(expr, lineno=expr.lineno)
 419
 420    def yield_expr(self, nodelist):
 421        if len(nodelist) > 1:
 422            value = self.com_node(nodelist[1])
 423        else:
 424            value = Const(None)
 425        return Yield(value, lineno=nodelist[0][2])
 426
 427    def raise_stmt(self, nodelist):
 428        # raise: [test [',' test [',' test]]]
 429        if len(nodelist) > 5:
 430            expr3 = self.com_node(nodelist[5])
 431        else:
 432            expr3 = None
 433        if len(nodelist) > 3:
 434            expr2 = self.com_node(nodelist[3])
 435        else:
 436            expr2 = None
 437        if len(nodelist) > 1:
 438            expr1 = self.com_node(nodelist[1])
 439        else:
 440            expr1 = None
 441        return Raise(expr1, expr2, expr3, lineno=nodelist[0][2])
 442
 443    def import_stmt(self, nodelist):
 444        # import_stmt: import_name | import_from
 445        assert len(nodelist) == 1
 446        return self.com_node(nodelist[0])
 447
 448    def import_name(self, nodelist):
 449        # import_name: 'import' dotted_as_names
 450        return Import(self.com_dotted_as_names(nodelist[1]),
 451                      lineno=nodelist[0][2])
 452
 453    def import_from(self, nodelist):
 454        # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' |
 455        #    '(' import_as_names ')' | import_as_names)
 456        assert nodelist[0][1] == 'from'
 457        idx = 1
 458        while nodelist[idx][1] == '.':
 459            idx += 1
 460        level = idx - 1
 461        if nodelist[idx][0] == symbol.dotted_name:
 462            fromname = self.com_dotted_name(nodelist[idx])
 463            idx += 1
 464        else:
 465            fromname = ""
 466        assert nodelist[idx][1] == 'import'
 467        if nodelist[idx + 1][0] == token.STAR:
 468            return From(fromname, [('*', None)], level,
 469                        lineno=nodelist[0][2])
 470        else:
 471            node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)]
 472            return From(fromname, self.com_import_as_names(node), level,
 473                        lineno=nodelist[0][2])
 474
 475    def global_stmt(self, nodelist):
 476        # global: NAME (',' NAME)*
 477        names = []
 478        for i in range(1, len(nodelist), 2):
 479            names.append(nodelist[i][1])
 480        return Global(names, lineno=nodelist[0][2])
 481
 482    def exec_stmt(self, nodelist):
 483        # exec_stmt: 'exec' expr ['in' expr [',' expr]]
 484        expr1 = self.com_node(nodelist[1])
 485        if len(nodelist) >= 4:
 486            expr2 = self.com_node(nodelist[3])
 487            if len(nodelist) >= 6:
 488                expr3 = self.com_node(nodelist[5])
 489            else:
 490                expr3 = None
 491        else:
 492            expr2 = expr3 = None
 493
 494        return Exec(expr1, expr2, expr3, lineno=nodelist[0][2])
 495
 496    def assert_stmt(self, nodelist):
 497        # 'assert': test, [',' test]
 498        expr1 = self.com_node(nodelist[1])
 499        if (len(nodelist) == 4):
 500            expr2 = self.com_node(nodelist[3])
 501        else:
 502            expr2 = None
 503        return Assert(expr1, expr2, lineno=nodelist[0][2])
 504
 505    def if_stmt(self, nodelist):
 506        # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 507        tests = []
 508        for i in range(0, len(nodelist) - 3, 4):
 509            testNode = self.com_node(nodelist[i + 1])
 510            suiteNode = self.com_node(nodelist[i + 3])
 511            tests.append((testNode, suiteNode))
 512
 513        if len(nodelist) % 4 == 3:
 514            elseNode = self.com_node(nodelist[-1])
 515##      elseNode.lineno = nodelist[-1][1][2]
 516        else:
 517            elseNode = None
 518        return If(tests, elseNode, lineno=nodelist[0][2])
 519
 520    def while_stmt(self, nodelist):
 521        # 'while' test ':' suite ['else' ':' suite]
 522
 523        testNode = self.com_node(nodelist[1])
 524        bodyNode = self.com_node(nodelist[3])
 525
 526        if len(nodelist) > 4:
 527            elseNode = self.com_node(nodelist[6])
 528        else:
 529            elseNode = None
 530
 531        return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])
 532
 533    def for_stmt(self, nodelist):
 534        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
 535
 536        assignNode = self.com_assign(nodelist[1], OP_ASSIGN)
 537        listNode = self.com_node(nodelist[3])
 538        bodyNode = self.com_node(nodelist[5])
 539
 540        if len(nodelist) > 8:
 541            elseNode = self.com_node(nodelist[8])
 542        else:
 543            elseNode = None
 544
 545        return For(assignNode, listNode, bodyNode, elseNode,
 546                   lineno=nodelist[0][2])
 547
 548    def try_stmt(self, nodelist):
 549        return self.com_try_except_finally(nodelist)
 550
 551    def with_stmt(self, nodelist):
 552        return self.com_with(nodelist)
 553
 554    def with_var(self, nodelist):
 555        return self.com_with_var(nodelist)
 556
 557    def suite(self, nodelist):
 558        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
 559        if len(nodelist) == 1:
 560            return self.com_stmt(nodelist[0])
 561
 562        stmts = []
 563        for node in nodelist:
 564            if node[0] == symbol.stmt:
 565                self.com_append_stmt(stmts, node)
 566        return Stmt(stmts)
 567
 568    # --------------------------------------------------------------
 569    #
 570    # EXPRESSION NODES  (invoked by com_node())
 571    #
 572
 573    def testlist(self, nodelist):
 574        # testlist: expr (',' expr)* [',']
 575        # testlist_safe: test [(',' test)+ [',']]
 576        # exprlist: expr (',' expr)* [',']
 577        return self.com_binary(Tuple, nodelist)
 578
 579    testlist_safe = testlist # XXX
 580    testlist1 = testlist
 581    exprlist = testlist
 582
 583    def testlist_gexp(self, nodelist):
 584        if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for:
 585            test = self.com_node(nodelist[0])
 586            return self.com_generator_expression(test, nodelist[1])
 587        return self.testlist(nodelist)
 588
 589    def test(self, nodelist):
 590        # or_test ['if' or_test 'else' test] | lambdef
 591        if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
 592            return self.lambdef(nodelist[0])
 593        then = self.com_node(nodelist[0])
 594        if len(nodelist) > 1:
 595            assert len(nodelist) == 5
 596            assert nodelist[1][1] == 'if'
 597            assert nodelist[3][1] == 'else'
 598            test = self.com_node(nodelist[2])
 599            else_ = self.com_node(nodelist[4])
 600            return IfExp(test, then, else_, lineno=nodelist[1][2])
 601        return then
 602
 603    def or_test(self, nodelist):
 604        # and_test ('or' and_test)* | lambdef
 605        if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
 606            return self.lambdef(nodelist[0])
 607        return self.com_binary(Or, nodelist)
 608    old_test = or_test
 609
 610    def and_test(self, nodelist):
 611        # not_test ('and' not_test)*
 612        return self.com_binary(And, nodelist)
 613
 614    def not_test(self, nodelist):
 615        # 'not' not_test | comparison
 616        result = self.com_node(nodelist[-1])
 617        if len(nodelist) == 2:
 618            return Not(result, lineno=nodelist[0][2])
 619        return result
 620
 621    def comparison(self, nodelist):
 622        # comparison: expr (comp_op expr)*
 623        node = self.com_node(nodelist[0])
 624        if len(nodelist) == 1:
 625            return node
 626
 627        results = []
 628        for i in range(2, len(nodelist), 2):
 629            nl = nodelist[i-1]
 630
 631            # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
 632            #          | 'in' | 'not' 'in' | 'is' | 'is' 'not'
 633            n = nl[1]
 634            if n[0] == token.NAME:
 635                type = n[1]
 636                if len(nl) == 3:
 637                    if type == 'not':
 638                        type = 'not in'
 639                    else:
 640                        type = 'is not'
 641            else:
 642                type = _cmp_types[n[0]]
 643
 644            lineno = nl[1][2]
 645            results.append((type, self.com_node(nodelist[i])))
 646
 647        # we need a special "compare" node so that we can distinguish
 648        #   3 < x < 5   from    (3 < x) < 5
 649        # the two have very different semantics and results (note that the
 650        # latter form is always true)
 651
 652        return Compare(node, results, lineno=lineno)
 653
 654    def expr(self, nodelist):
 655        # xor_expr ('|' xor_expr)*
 656        return self.com_binary(Bitor, nodelist)
 657
 658    def xor_expr(self, nodelist):
 659        # xor_expr ('^' xor_expr)*
 660        return self.com_binary(Bitxor, nodelist)
 661
 662    def and_expr(self, nodelist):
 663        # xor_expr ('&' xor_expr)*
 664        return self.com_binary(Bitand, nodelist)
 665
 666    def shift_expr(self, nodelist):
 667        # shift_expr ('<<'|'>>' shift_expr)*
 668        node = self.com_node(nodelist[0])
 669        for i in range(2, len(nodelist), 2):
 670            right = self.com_node(nodelist[i])
 671            if nodelist[i-1][0] == token.LEFTSHIFT:
 672                node = LeftShift([node, right], lineno=nodelist[1][2])
 673            elif nodelist[i-1][0] == token.RIGHTSHIFT:
 674                node = RightShift([node, right], lineno=nodelist[1][2])
 675            else:
 676                raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
 677        return node
 678
 679    def arith_expr(self, nodelist):
 680        node = self.com_node(nodelist[0])
 681        for i in range(2, len(nodelist), 2):
 682            right = self.com_node(nodelist[i])
 683            if nodelist[i-1][0] == token.PLUS:
 684                node = Add([node, right], lineno=nodelist[1][2])
 685            elif nodelist[i-1][0] == token.MINUS:
 686                node = Sub([node, right], lineno=nodelist[1][2])
 687            else:
 688                raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
 689        return node
 690
 691    def term(self, nodelist):
 692        node = self.com_node(nodelist[0])
 693        for i in range(2, len(nodelist), 2):
 694            right = self.com_node(nodelist[i])
 695            t = nodelist[i-1][0]
 696            if t == token.STAR:
 697                node = Mul([node, right])
 698            elif t == token.SLASH:
 699                node = Div([node, right])
 700            elif t == token.PERCENT:
 701                node = Mod([node, right])
 702            elif t == token.DOUBLESLASH:
 703                node = FloorDiv([node, right])
 704            else:
 705                raise ValueError, "unexpected token: %s" % t
 706            node.lineno = nodelist[1][2]
 707        return node
 708
 709    def factor(self, nodelist):
 710        elt = nodelist[0]
 711        t = elt[0]
 712        node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
 713        # need to handle (unary op)constant here...
 714        if t == token.PLUS:
 715            return UnaryAdd(node, lineno=elt[2])
 716        elif t == token.MINUS:
 717            return UnarySub(node, lineno=elt[2])
 718        elif t == token.TILDE:
 719            node = Invert(node, lineno=elt[2])
 720        return node
 721
 722    def power(self, nodelist):
 723        # power: atom trailer* ('**' factor)*
 724        node = self.com_node(nodelist[0])
 725        for i in range(1, len(nodelist)):
 726            elt = nodelist[i]
 727            if elt[0] == token.DOUBLESTAR:
 728                return Power([node, self.com_node(nodelist[i+1])],
 729                             lineno=elt[2])
 730
 731            node = self.com_apply_trailer(node, elt)
 732
 733        return node
 734
 735    def atom(self, nodelist):
 736        return self._atom_dispatch[nodelist[0][0]](nodelist)
 737
 738    def atom_lpar(self, nodelist):
 739        if nodelist[1][0] == token.RPAR:
 740            return Tuple((), lineno=nodelist[0][2])
 741        return self.com_node(nodelist[1])
 742
 743    def atom_lsqb(self, nodelist):
 744        if nodelist[1][0] == token.RSQB:
 745            return List((), lineno=nodelist[0][2])
 746        return self.com_list_constructor(nodelist[1])
 747
 748    def atom_lbrace(self, nodelist):
 749        if nodelist[1][0] == token.RBRACE:
 750            return Dict((), lineno=nodelist[0][2])
 751        return self.com_dictmaker(nodelist[1])
 752
 753    def atom_backquote(self, nodelist):
 754        return Backquote(self.com_node(nodelist[1]))
 755
 756    def atom_number(self, nodelist):
 757        ### need to verify this matches compile.c
 758        k = eval(nodelist[0][1])
 759        return Const(k, lineno=nodelist[0][2])
 760
 761    def decode_literal(self, lit):
 762        if self.encoding:
 763            # this is particularly fragile & a bit of a
 764            # hack... changes in compile.c:parsestr and
 765            # tokenizer.c must be reflected here.
 766            if self.encoding not in ['utf-8', 'iso-8859-1']:
 767                lit = unicode(lit, 'utf-8').encode(self.encoding)
 768            return eval("# coding: %s\n%s" % (self.encoding, lit))
 769        else:
 770            return eval(lit)
 771
 772    def atom_string(self, nodelist):
 773        k = ''
 774        for node in nodelist:
 775            k += self.decode_literal(node[1])
 776        return Const(k, lineno=nodelist[0][2])
 777
 778    def atom_name(self, nodelist):
 779        return Name(nodelist[0][1], lineno=nodelist[0][2])
 780
 781    # --------------------------------------------------------------
 782    #
 783    # INTERNAL PARSING UTILITIES
 784    #
 785
 786    # The use of com_node() introduces a lot of extra stack frames,
 787    # enough to cause a stack overflow compiling test.test_parser with
 788    # the standard interpreter recursionlimit.  The com_node() is a
 789    # convenience function that hides the dispatch details, but comes
 790    # at a very high cost.  It is more efficient to dispatch directly
 791    # in the callers.  In these cases, use lookup_node() and call the
 792    # dispatched node directly.
 793
 794    def lookup_node(self, node):
 795        return self._dispatch[node[0]]
 796
 797    def com_node(self, node):
 798        # Note: compile.c has handling in com_node for del_stmt, pass_stmt,
 799        #       break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
 800        #       and compound_stmt.
 801        #       We'll just dispatch them.
 802        return self._dispatch[node[0]](node[1:])
 803
 804    def com_NEWLINE(self, *args):
 805        # A ';' at the end of a line can make a NEWLINE token appear
 806        # here, Render it harmless. (genc discards ('discard',
 807        # ('const', xxxx)) Nodes)
 808        return Discard(Const(None))
 809
 810    def com_arglist(self, nodelist):
 811        # varargslist:
 812        #     (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME)
 813        #   | fpdef ['=' test] (',' fpdef ['=' test])* [',']
 814        # fpdef: NAME | '(' fplist ')'
 815        # fplist: fpdef (',' fpdef)* [',']
 816        names = []
 817        defaults = []
 818        flags = 0
 819
 820        i = 0
 821        while i < len(nodelist):
 822            node = nodelist[i]
 823            if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
 824                if node[0] == token.STAR:
 825                    node = nodelist[i+1]
 826                    if node[0] == token.NAME:
 827                        names.append(node[1])
 828                        flags = flags | CO_VARARGS
 829                        i = i + 3
 830
 831                if i < len(nodelist):
 832                    # should be DOUBLESTAR
 833                    t = nodelist[i][0]
 834                    if t == token.DOUBLESTAR:
 835                        node = nodelist[i+1]
 836                    else:
 837                        raise ValueError, "unexpected token: %s" % t
 838                    names.append(node[1])
 839                    flags = flags | CO_VARKEYWORDS
 840
 841                break
 842
 843            # fpdef: NAME | '(' fplist ')'
 844            names.append(self.com_fpdef(node))
 845
 846            i = i + 1
 847            if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
 848                defaults.append(self.com_node(nodelist[i + 1]))
 849                i = i + 2
 850            elif len(defaults):
 851                # we have already seen an argument with default, but here
 852                # came one without
 853                raise SyntaxError, "non-default argument follows default argument"
 854
 855            # skip the comma
 856            i = i + 1
 857
 858        return names, defaults, flags
 859
 860    def com_fpdef(self, node):
 861        # fpdef: NAME | '(' fplist ')'
 862        if node[1][0] == token.LPAR:
 863            return self.com_fplist(node[2])
 864        return node[1][1]
 865
 866    def com_fplist(self, node):
 867        # fplist: fpdef (',' fpdef)* [',']
 868        if len(node) == 2:
 869            return self.com_fpdef(node[1])
 870        list = []
 871        for i in range(1, len(node), 2):
 872            list.append(self.com_fpdef(node[i]))
 873        return tuple(list)
 874
 875    def com_dotted_name(self, node):
 876        # String together the dotted names and return the string
 877        name = ""
 878        for n in node:
 879            if type(n) == type(()) and n[0] == 1:
 880                name = name + n[1] + '.'
 881        return name[:-1]
 882
 883    def com_dotted_as_name(self, node):
 884        assert node[0] == symbol.dotted_as_name
 885        node = node[1:]
 886        dot = self.com_dotted_name(node[0][1:])
 887        if len(node) == 1:
 888            return dot, None
 889        assert node[1][1] == 'as'
 890        assert node[2][0] == token.NAME
 891        return dot, node[2][1]
 892
 893    def com_dotted_as_names(self, node):
 894        assert node[0] == symbol.dotted_as_names
 895        node = node[1:]
 896        names = [self.com_dotted_as_name(node[0])]
 897        for i in range(2, len(node), 2):
 898            names.append(self.com_dotted_as_name(node[i]))
 899        return names
 900
 901    def com_import_as_name(self, node):
 902        assert node[0] == symbol.import_as_name
 903        node = node[1:]
 904        assert node[0][0] == token.NAME
 905        if len(node) == 1:
 906            return node[0][1], None
 907        assert node[1][1] == 'as', node
 908        assert node[2][0] == token.NAME
 909        return node[0][1], node[2][1]
 910
 911    def com_import_as_names(self, node):
 912        assert node[0] == symbol.import_as_names
 913        node = node[1:]
 914        names = [self.com_import_as_name(node[0])]
 915        for i in range(2, len(node), 2):
 916            names.append(self.com_import_as_name(node[i]))
 917        return names
 918
 919    def com_bases(self, node):
 920        bases = []
 921        for i in range(1, len(node), 2):
 922            bases.append(self.com_node(node[i]))
 923        return bases
 924
 925    def com_try_except_finally(self, nodelist):
 926        # ('try' ':' suite
 927        #  ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite]
 928        #   | 'finally' ':' suite))
 929
 930        if nodelist[3][0] == token.NAME:
 931            # first clause is a finally clause: only try-finally
 932            return TryFinally(self.com_node(nodelist[2]),
 933                              self.com_node(nodelist[5]),
 934                              lineno=nodelist[0][2])
 935
 936        #tryexcept:  [TryNode, [except_clauses], elseNode)]
 937        clauses = []
 938        elseNode = None
 939        finallyNode = None
 940        for i in range(3, len(nodelist), 3):
 941            node = nodelist[i]
 942            if node[0] == symbol.except_clause:
 943                # except_clause: 'except' [expr [(',' | 'as') expr]] */
 944                if len(node) > 2:
 945                    expr1 = self.com_node(node[2])
 946                    if len(node) > 4:
 947                        expr2 = self.com_assign(node[4], OP_ASSIGN)
 948                    else:
 949                        expr2 = None
 950                else:
 951                    expr1 = expr2 = None
 952                clauses.append((expr1, expr2, self.com_node(nodelist[i+2])))
 953
 954            if node[0] == token.NAME:
 955                if node[1] == 'else':
 956                    elseNode = self.com_node(nodelist[i+2])
 957                elif node[1] == 'finally':
 958                    finallyNode = self.com_node(nodelist[i+2])
 959        try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode,
 960                               lineno=nodelist[0][2])
 961        if finallyNode:
 962            return TryFinally(try_except, finallyNode, lineno=nodelist[0][2])
 963        else:
 964            return try_except
 965
 966    def com_with(self, nodelist):
 967        # with_stmt: 'with' expr [with_var] ':' suite
 968        expr = self.com_node(nodelist[1])
 969        body = self.com_node(nodelist[-1])
 970        if nodelist[2][0] == token.COLON:
 971            var = None
 972        else:
 973            var = self.com_assign(nodelist[2][2], OP_ASSIGN)
 974        return With(expr, var, body, lineno=nodelist[0][2])
 975
 976    def com_with_var(self, nodelist):
 977        # with_var: 'as' expr
 978        return self.com_node(nodelist[1])
 979
 980    def com_augassign_op(self, node):
 981        assert node[0] == symbol.augassign
 982        return node[1]
 983
 984    def com_augassign(self, node):
 985        """Return node suitable for lvalue of augmented assignment
 986
 987        Names, slices, and attributes are the only allowable nodes.
 988        """
 989        l = self.com_node(node)
 990        if l.__class__ in (Name, Slice, Subscript, Getattr):
 991            return l
 992        raise SyntaxError, "can't assign to %s" % l.__class__.__name__
 993
 994    def com_assign(self, node, assigning):
 995        # return a node suitable for use as an "lvalue"
 996        # loop to avoid trivial recursion
 997        while 1:
 998            t = node[0]
 999            if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_gexp):
1000                if len(node) > 2:
1001                    return self.com_assign_tuple(node, assigning)
1002                node = node[1]
1003            elif t in _assign_types:
1004                if len(node) > 2:
1005                    raise SyntaxError, "can't assign to operator"
1006                node = node[1]
1007            elif t == symbol.power:
1008                if node[1][0] != symbol.atom:
1009                    raise SyntaxError, "can't assign to operator"
1010                if len(node) > 2:
1011                    primary = self.com_node(node[1])
1012                    for i in range(2, len(node)-1):
1013                        ch = node[i]
1014                        if ch[0] == token.DOUBLESTAR:
1015                            raise SyntaxError, "can't assign to operator"
1016                        primary = self.com_apply_trailer(primary, ch)
1017                    return self.com_assign_trailer(primary, node[-1],
1018                                                   assigning)
1019                node = node[1]
1020            elif t == symbol.atom:
1021                t = node[1][0]
1022                if t == token.LPAR:
1023                    node = node[2]
1024                    if node[0] == token.RPAR:
1025                        raise SyntaxError, "can't assign to ()"
1026                elif t == token.LSQB:
1027                    node = node[2]
1028                    if node[0] == token.RSQB:
1029                        raise SyntaxError, "can't assign to []"
1030                    return self.com_assign_list(node, assigning)
1031                elif t == token.NAME:
1032                    return self.com_assign_name(node[1], assigning)
1033                else:
1034                    raise SyntaxError, "can't assign to literal"
1035            else:
1036                raise SyntaxError, "bad assignment (%s)" % t
1037
1038    def com_assign_tuple(self, node, assigning):
1039        assigns = []
1040        for i in range(1, len(node), 2):
1041            assigns.append(self.com_assign(node[i], assigning))
1042        return AssTuple(assigns, lineno=extractLineNo(node))
1043
1044    def com_assign_list(self, node, assigning):
1045        assigns = []
1046        for i in range(1, len(node), 2):
1047            if i + 1 < len(node):
1048                if node[i + 1][0] == symbol.list_for:
1049                    raise SyntaxError, "can't assign to list comprehension"
1050                assert node[i + 1][0] == token.COMMA, node[i + 1]
1051            assigns.append(self.com_assign(node[i], assigning))
1052        return AssList(assigns, lineno=extractLineNo(node))
1053
1054    def com_assign_name(self, node, assigning):
1055        return AssName(node[1], assigning, lineno=node[2])
1056
1057    def com_assign_trailer(self, primary, node, assigning):
1058        t = node[1][0]
1059        if t == token.DOT:
1060            return self.com_assign_attr(primary, node[2], assigning)
1061        if t == token.LSQB:
1062            return self.com_subscriptlist(primary, node[2], assigning)
1063        if t == token.LPAR:
1064            raise SyntaxError, "can't assign to function call"
1065        raise SyntaxError, "unknown trailer type: %s" % t
1066
1067    def com_assign_attr(self, primary, node, assigning):
1068        return AssAttr(primary, node[1], assigning, lineno=node[-1])
1069
1070    def com_binary(self, constructor, nodelist):
1071        "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
1072        l = len(nodelist)
1073        if l == 1:
1074            n = nodelist[0]
1075            return self.lookup_node(n)(n[1:])
1076        items = []
1077        for i in range(0, l, 2):
1078            n = nodelist[i]
1079            items.append(self.lookup_node(n)(n[1:]))
1080        return constructor(items, lineno=extractLineNo(nodelist))
1081
1082    def com_stmt(self, node):
1083        result = self.lookup_node(node)(node[1:])
1084        assert result is not None
1085        if isinstance(result, Stmt):
1086            return result
1087        return Stmt([result])
1088
1089    def com_append_stmt(self, stmts, node):
1090        result = self.lookup_node(node)(node[1:])
1091        assert result is not None
1092        if isinstance(result, Stmt):
1093            stmts.extend(result.nodes)
1094        else:
1095            stmts.append(result)
1096
1097    if hasattr(symbol, 'list_for'):
1098        def com_list_constructor(self, nodelist):
1099            # listmaker: test ( list_for | (',' test)* [','] )
1100            values = []
1101            for i in range(1, len(nodelist)):
1102                if nodelist[i][0] == symbol.list_for:
1103                    assert len(nodelist[i:]) == 1
1104                    return self.com_list_comprehension(values[0],
1105                                                       nodelist[i])
1106                elif nodelist[i][0] == token.COMMA:
1107                    continue
1108                values.append(self.com_node(nodelist[i]))
1109            return List(values, lineno=values[0].lineno)
1110
1111        def com_list_comprehension(self, expr, node):
1112            # list_iter: list_for | list_if
1113            # list_for: 'for' exprlist 'in' testlist [list_iter]
1114            # list_if: 'if' test [list_iter]
1115
1116            # XXX should raise SyntaxError for assignment
1117
1118            lineno = node[1][2]
1119            fors = []
1120            while node:
1121                t = node[1][1]
1122                if t == 'for':
1123                    assignNode = self.com_assign(node[2], OP_ASSIGN)
1124                    listNode = self.com_node(node[4])
1125                    newfor = ListCompFor(assignNode, listNode, [])
1126                    newfor.lineno = node[1][2]
1127                    fors.append(newfor)
1128                    if len(node) == 5:
1129                        node = None
1130                    else:
1131                        node = self.com_list_iter(node[5])
1132                elif t == 'if':
1133                    test = self.com_node(node[2])
1134                    newif = ListCompIf(test, lineno=node[1][2])
1135                    newfor.ifs.append(newif)
1136                    if len(node) == 3:
1137                        node = None
1138                    else:
1139                        node = self.com_list_iter(node[3])
1140                else:
1141                    raise SyntaxError, \
1142                          ("unexpected list comprehension element: %s %d"
1143                           % (node, lineno))
1144            return ListComp(expr, fors, lineno=lineno)
1145
1146        def com_list_iter(self, node):
1147            assert node[0] == symbol.list_iter
1148            return node[1]
1149    else:
1150        def com_list_constructor(self, nodelist):
1151            values = []
1152            for i in range(1, len(nodelist), 2):
1153                values.append(self.com_node(nodelist[i]))
1154            return List(values, lineno=values[0].lineno)
1155
1156    if hasattr(symbol, 'gen_for'):
1157        def com_generator_expression(self, expr, node):
1158            # gen_iter: gen_for | gen_if
1159            # gen_for: 'for' exprlist 'in' test [gen_iter]
1160            # gen_if: 'if' test [gen_iter]
1161
1162            lineno = node[1][2]
1163            fors = []
1164            while node:
1165                t = node[1][1]
1166                if t == 'for':
1167                    assignNode = self.com_assign(node[2], OP_ASSIGN)
1168                    genNode = self.com_node(node[4])
1169                    newfor = GenExprFor(assignNode, genNode, [],
1170                                        lineno=node[1][2])
1171                    fors.append(newfor)
1172                    if (len(node)) == 5:
1173                        node = None
1174                    else:
1175                        node = self.com_gen_iter(node[5])
1176                elif t == 'if':
1177                    test = self.com_node(node[2])
1178                    newif = GenExprIf(test, lineno=node[1][2])
1179                    newfor.ifs.append(newif)
1180                    if len(node) == 3:
1181                        node = None
1182                    else:
1183                        node = self.com_gen_iter(node[3])
1184                else:
1185                    raise SyntaxError, \
1186                            ("unexpected generator expression element: %s %d"
1187                             % (node, lineno))
1188            fors[0].is_outmost = True
1189            return GenExpr(GenExprInner(expr, fors), lineno=lineno)
1190
1191        def com_gen_iter(self, node):
1192            assert node[0] == symbol.gen_iter
1193            return node[1]
1194
1195    def com_dictmaker(self, nodelist):
1196        # dictmaker: test ':' test (',' test ':' value)* [',']
1197        items = []
1198        for i in range(1, len(nodelist), 4):
1199            items.append((self.com_node(nodelist[i]),
1200                          self.com_node(nodelist[i+2])))
1201        return Dict(items, lineno=items[0][0].lineno)
1202
1203    def com_apply_trailer(self, primaryNode, nodelist):
1204        t = nodelist[1][0]
1205        if t == token.LPAR:
1206            return self.com_call_function(primaryNode, nodelist[2])
1207        if t == token.DOT:
1208            return self.com_select_member(primaryNode, nodelist[2])
1209        if t == token.LSQB:
1210            return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY)
1211
1212        raise SyntaxError, 'unknown node type: %s' % t
1213
1214    def com_select_member(self, primaryNode, nodelist):
1215        if nodelist[0] != token.NAME:
1216            raise SyntaxError, "member must be a name"
1217        return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
1218
1219    def com_call_function(self, primaryNode, nodelist):
1220        if nodelist[0] == token.RPAR:
1221            return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
1222        args = []
1223        kw = 0
1224        star_node = dstar_node = None
1225        len_nodelist = len(nodelist)
1226        i = 1
1227        while i < len_nodelist:
1228            node = nodelist[i]
1229
1230            if node[0]==token.STAR:
1231                if star_node is not None:
1232                    raise SyntaxError, 'already have the varargs indentifier'
1233                star_node = self.com_node(nodelist[i+1])
1234                i = i + 3
1235                continue
1236            elif node[0]==token.DOUBLESTAR:
1237                if dstar_node is not None:
1238                    raise SyntaxError, 'already have the kwargs indentifier'
1239                dstar_node = self.com_node(nodelist[i+1])
1240                i = i + 3
1241                continue
1242
1243            # positional or named parameters
1244            kw, result = self.com_argument(node, kw, star_node)
1245
1246            if len_nodelist != 2 and isinstance(result, GenExpr) \
1247               and len(node) == 3 and node[2][0] == symbol.gen_for:
1248                # allow f(x for x in y), but reject f(x for x in y, 1)
1249                # should use f((x for x in y), 1) instead of f(x for x in y, 1)
1250                raise SyntaxError, 'generator expression needs parenthesis'
1251
1252            args.append(result)
1253            i = i + 2
1254
1255        return CallFunc(primaryNode, args, star_node, dstar_node,
1256                        lineno=extractLineNo(nodelist))
1257
1258    def com_argument(self, nodelist, kw, star_node):
1259        if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for:
1260            test = self.com_node(nodelist[1])
1261            return 0, self.com_generator_expression(test, nodelist[2])
1262        if len(nodelist) == 2:
1263            if kw:
1264                raise SyntaxError, "non-keyword arg after keyword arg"
1265            if star_node:
1266                raise SyntaxError, "only named arguments may follow *expression"
1267            return 0, self.com_node(nodelist[1])
1268        result = self.com_node(nodelist[3])
1269        n = nodelist[1]
1270        while len(n) == 2 and n[0] != token.NAME:
1271            n = n[1]
1272        if n[0] != token.NAME:
1273            raise SyntaxError, "keyword can't be an expression (%s)"%n[0]
1274        node = Keyword(n[1], result, lineno=n[2])
1275        return 1, node
1276
1277    def com_subscriptlist(self, primary, nodelist, assigning):
1278        # slicing:      simple_slicing | extended_slicing
1279        # simple_slicing:   primary "[" short_slice "]"
1280        # extended_slicing: primary "[" slice_list "]"
1281        # slice_list:   slice_item ("," slice_item)* [","]
1282
1283        # backwards compat slice for '[i:j]'
1284        if len(nodelist) == 2:
1285            sub = nodelist[1]
1286            if (sub[1][0] == token.COLON or \
1287                            (len(sub) > 2 and sub[2][0] == token.COLON)) and \
1288                            sub[-1][0] != symbol.sliceop:
1289                return self.com_slice(primary, sub, assigning)
1290
1291        subscripts = []
1292        for i in range(1, len(nodelist), 2):
1293            subscripts.append(self.com_subscript(nodelist[i]))
1294        return Subscript(primary, assigning, subscripts,
1295                         lineno=extractLineNo(nodelist))
1296
1297    def com_subscript(self, node):
1298        # slice_item: expression | proper_slice | ellipsis
1299        ch = node[1]
1300        t = ch[0]
1301        if t == token.DOT and node[2][0] == token.DOT:
1302            return Ellipsis()
1303        if t == token.COLON or len(node) > 2:
1304            return self.com_sliceobj(node)
1305        return self.com_node(ch)
1306
1307    def com_sliceobj(self, node):
1308        # proper_slice: short_slice | long_slice
1309        # short_slice:  [lower_bound] ":" [upper_bound]
1310        # long_slice:   short_slice ":" [stride]
1311        # lower_bound:  expression
1312        # upper_bound:  expression
1313        # stride:       expression
1314        #
1315        # Note: a stride may be further slicing...
1316
1317        items = []
1318
1319        if node[1][0] == token.COLON:
1320            items.append(Const(None))
1321            i = 2
1322        else:
1323            items.append(self.com_node(node[1]))
1324            # i == 2 is a COLON
1325            i = 3
1326
1327        if i < len(node) and node[i][0] == symbol.test:
1328            items.append(self.com_node(node[i]))
1329            i = i + 1
1330        else:
1331            items.append(Const(None))
1332
1333        # a short_slice has been built. look for long_slice now by looking
1334        # for strides...
1335        for j in range(i, len(node)):
1336            ch = node[j]
1337            if len(ch) == 2:
1338                items.append(Const(None))
1339            else:
1340                items.append(self.com_node(ch[2]))
1341        return Sliceobj(items, lineno=extractLineNo(node))
1342
1343    def com_slice(self, primary, node, assigning):
1344        # short_slice:  [lower_bound] ":" [upper_bound]
1345        lower = upper = None
1346        if len(node) == 3:
1347            if node[1][0] == token.COLON:
1348                upper = self.com_node(node[2])
1349            else:
1350                lower = self.com_node(node[1])
1351        elif len(node) == 4:
1352            lower = self.com_node(node[1])
1353            upper = self.com_node(node[3])
1354        return Slice(primary, assigning, lower, upper,
1355                     lineno=extractLineNo(node))
1356
1357    def get_docstring(self, node, n=None):
1358        if n is None:
1359            n = node[0]
1360            node = node[1:]
1361        if n == symbol.suite:
1362            if len(node) == 1:
1363                return self.get_docstring(node[0])
1364            for sub in node:
1365                if sub[0] == symbol.stmt:
1366                    return self.get_docstring(sub)
1367            return None
1368        if n == symbol.file_input:
1369            for sub in node:
1370                if sub[0] == symbol.stmt:
1371                    return self.get_docstring(sub)
1372            return None
1373        if n == symbol.atom:
1374            if node[0][0] == token.STRING:
1375                s = ''
1376                for t in node:
1377                    s = s + eval(t[1])
1378                return s
1379            return None
1380        if n == symbol.stmt or n == symbol.simple_stmt \
1381           or n == symbol.small_stmt:
1382            return self.get_docstring(node[0])
1383        if n in _doc_nodes and len(node) == 1:
1384            return self.get_docstring(node[0])
1385        return None
1386
1387
1388_doc_nodes = [
1389    symbol.expr_stmt,
1390    symbol.testlist,
1391    symbol.testlist_safe,
1392    symbol.test,
1393    symbol.or_test,
1394    symbol.and_test,
1395    symbol.not_test,
1396    symbol.comparison,
1397    symbol.expr,
1398    symbol.xor_expr,
1399    symbol.and_expr,
1400    symbol.shift_expr,
1401    symbol.arith_expr,
1402    symbol.term,
1403    symbol.factor,
1404    symbol.power,
1405    ]
1406
1407# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
1408#             | 'in' | 'not' 'in' | 'is' | 'is' 'not'
1409_cmp_types = {
1410    token.LESS : '<',
1411    token.GREATER : '>',
1412    token.EQEQUAL : '==',
1413    token.EQUAL : '==',
1414    token.LESSEQUAL : '<=',
1415    token.GREATEREQUAL : '>=',
1416    token.NOTEQUAL : '!=',
1417    }
1418
1419_legal_node_types = [
1420    symbol.funcdef,
1421    symbol.classdef,
1422    symbol.stmt,
1423    symbol.small_stmt,
1424    symbol.flow_stmt,
1425    symbol.simple_stmt,
1426 

Large files files are truncated, but you can click here to view the full file