/DICK.B1/IronPython/Compiler/Parser.cs

Large files files are truncated, but you can click here to view the full file

/* ****************************************************************************

 *

 * Copyright (c) Microsoft Corporation. 

 *

 * This source code is subject to terms and conditions of the Microsoft Public License. A 

 * copy of the license can be found in the License.html file at the root of this distribution. If 

 * you cannot locate the  Microsoft Public License, please send an email to 

 * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 

 * by the terms of the Microsoft Public License.

 *

 * You must not remove this notice, or any other, from this software.

 *

 *

 * ***************************************************************************/



using System;

using System.Collections.Generic;

using System.Diagnostics;

using System.IO;



using Microsoft.Scripting;

using Microsoft.Scripting.Runtime;

using Microsoft.Scripting.Utils;



using IronPython.Compiler.Ast;

using IronPython.Hosting;

using IronPython.Runtime;

using IronPython.Runtime.Types;



#if CLR2

using Microsoft.Scripting.Math;

#else

using System.Numerics;

#endif



namespace IronPython.Compiler {



    public class Parser : IDisposable { // TODO: remove IDisposable

        // immutable properties:

        private readonly Tokenizer _tokenizer;



        // mutable properties:

        private ErrorSink _errors;

        private ParserSink _sink;



        // resettable properties:

        private SourceUnit _sourceUnit;



        /// <summary>

        /// Language features initialized on parser construction and possibly updated during parsing. 

        /// The code can set the language features (e.g. "from __future__ import division").

        /// </summary>

        private ModuleOptions _languageFeatures;



        // state:

        private TokenWithSpan _token;

        private TokenWithSpan _lookahead;

        private Stack<FunctionDefinition> _functions;

        private bool _fromFutureAllowed;

        private string _privatePrefix;

        private bool _parsingStarted, _allowIncomplete;

        private bool _inLoop, _inFinally, _isGenerator, _returnWithValue;

        private SourceCodeReader _sourceReader;

        private int _errorCode;

        private readonly CompilerContext _context;



        private static readonly char[] newLineChar = new char[] { '\n' };

        private static readonly char[] whiteSpace = { ' ', '\t' };

      

        #region Construction



        private Parser(CompilerContext context, Tokenizer tokenizer, ErrorSink errorSink, ParserSink parserSink, ModuleOptions languageFeatures) {

            ContractUtils.RequiresNotNull(tokenizer, "tokenizer");

            ContractUtils.RequiresNotNull(errorSink, "errorSink");

            ContractUtils.RequiresNotNull(parserSink, "parserSink");



            tokenizer.ErrorSink = new TokenizerErrorSink(this);



            _tokenizer = tokenizer;

            _errors = errorSink;

            _sink = parserSink;

            _context = context;



            Reset(tokenizer.SourceUnit, languageFeatures);

        }



        public static Parser CreateParser(CompilerContext context, PythonOptions options) {

            return CreateParserWorker(context, options, false);

        }



        [Obsolete("pass verbatim via PythonCompilerOptions in PythonOptions")]

        public static Parser CreateParser(CompilerContext context, PythonOptions options, bool verbatim) {

            return CreateParserWorker(context, options, verbatim);

        }



        private static Parser CreateParserWorker(CompilerContext context, PythonOptions options, bool verbatim) {

            ContractUtils.RequiresNotNull(context, "context");

            ContractUtils.RequiresNotNull(options, "options");



            PythonCompilerOptions compilerOptions = context.Options as PythonCompilerOptions;

            if (options == null) {

                throw new ArgumentException(Resources.PythonContextRequired);

            }



            SourceCodeReader reader;



            try {

                reader = context.SourceUnit.GetReader();



                if (compilerOptions.SkipFirstLine) {

                    reader.ReadLine();

                }

            } catch (IOException e) {

                context.Errors.Add(context.SourceUnit, e.Message, SourceSpan.Invalid, 0, Severity.Error);

                throw;

            }



            Tokenizer tokenizer = new Tokenizer(context.Errors, compilerOptions, verbatim);

            tokenizer.Initialize(null, reader, context.SourceUnit, SourceLocation.MinValue);

            tokenizer.IndentationInconsistencySeverity = options.IndentationInconsistencySeverity;



            Parser result = new Parser(context, tokenizer, context.Errors, context.ParserSink, compilerOptions.Module);

            result._sourceReader = reader;

            return result;

        }



        #endregion



        #region Public parser interface



        public PythonAst ParseFile(bool makeModule) {

            return ParseFile(makeModule, false);

        }



        //single_input: Newline | simple_stmt | compound_stmt Newline

        //eval_input: testlist Newline* ENDMARKER

        //file_input: (Newline | stmt)* ENDMARKER

        public PythonAst ParseFile(bool makeModule, bool returnValue) {

            try {

                return ParseFileWorker(makeModule, returnValue);

            } catch (BadSourceException bse) {

                throw BadSourceError(bse);

            }

        }

        

        //[stmt_list] Newline | compound_stmt Newline

        //stmt_list ::= simple_stmt (";" simple_stmt)* [";"]

        //compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef

        //Returns a simple or coumpound_stmt or null if input is incomplete

        /// <summary>

        /// Parse one or more lines of interactive input

        /// </summary>

        /// <returns>null if input is not yet valid but could be with more lines</returns>

        public PythonAst ParseInteractiveCode(out ScriptCodeParseResult properties) {

            bool parsingMultiLineCmpdStmt;

            bool isEmptyStmt = false;



            properties = ScriptCodeParseResult.Complete;



            StartParsing();

            Statement ret = InternalParseInteractiveInput(out parsingMultiLineCmpdStmt, out isEmptyStmt);



            if (_errorCode == 0) {

                if (isEmptyStmt) {

                    properties = ScriptCodeParseResult.Empty;

                } else if (parsingMultiLineCmpdStmt) {

                    properties = ScriptCodeParseResult.IncompleteStatement;

                }



                if (isEmptyStmt) {

                    return null;

                }



                return new PythonAst(ret, false, _languageFeatures, true, _context);

            } else {

                if ((_errorCode & ErrorCodes.IncompleteMask) != 0) {

                    if ((_errorCode & ErrorCodes.IncompleteToken) != 0) {

                        properties = ScriptCodeParseResult.IncompleteToken;

                        return null;

                    }



                    if ((_errorCode & ErrorCodes.IncompleteStatement) != 0) {

                        if (parsingMultiLineCmpdStmt) {

                            properties = ScriptCodeParseResult.IncompleteStatement;

                        } else {

                            properties = ScriptCodeParseResult.IncompleteToken;

                        }

                        return null;

                    }

                }



                properties = ScriptCodeParseResult.Invalid;

                return null;

            }

        }



        public PythonAst ParseSingleStatement() {

            try {

                StartParsing();



                MaybeEatNewLine();

                Statement statement = ParseStmt();

                EatEndOfInput();

                return new PythonAst(statement, false, _languageFeatures, true, _context);

            } catch (BadSourceException bse) {

                throw BadSourceError(bse);

            }

        }



        public PythonAst ParseTopExpression() {

            try {

                // TODO: move from source unit  .TrimStart(' ', '\t')

                ReturnStatement ret = new ReturnStatement(ParseTestListAsExpression());

                ret.SetLoc(SourceSpan.None);

                return new PythonAst(ret, false, _languageFeatures, false, _context);

            } catch (BadSourceException bse) {

                throw BadSourceError(bse);

            }

        }



        /// <summary>

        /// Given the interactive text input for a compound statement, calculate what the

        /// indentation level of the next line should be

        /// </summary>

        public static int GetNextAutoIndentSize(string text, int autoIndentTabWidth) {

            ContractUtils.RequiresNotNull(text, "text");



            Debug.Assert(text[text.Length - 1] == '\n');

            string[] lines = text.Split(newLineChar);

            if (lines.Length <= 1) return 0;

            string lastLine = lines[lines.Length - 2];



            // Figure out the number of white-spaces at the start of the last line

            int startingSpaces = 0;

            while (startingSpaces < lastLine.Length && lastLine[startingSpaces] == ' ')

                startingSpaces++;



            // Assume the same indent as the previous line

            int autoIndentSize = startingSpaces;

            // Increase the indent if this looks like the start of a compounds statement.

            // Ideally, we would ask the parser to tell us the exact indentation level

            if (lastLine.TrimEnd(whiteSpace).EndsWith(":"))

                autoIndentSize += autoIndentTabWidth;



            return autoIndentSize;

        }



        public ErrorSink ErrorSink {

            get {

                return _errors;

            }

            set {

                ContractUtils.RequiresNotNull(value, "value");

                _errors = value;

            }

        }



        public ParserSink ParserSink {

            get {

                return _sink;

            }

            set {

                ContractUtils.RequiresNotNull(value, "value");

                _sink = value;

            }

        }



        public int ErrorCode {

            get { return _errorCode; }

        }



        public void Reset(SourceUnit sourceUnit, ModuleOptions languageFeatures) {

            ContractUtils.RequiresNotNull(sourceUnit, "sourceUnit");



            _sourceUnit = sourceUnit;

            _languageFeatures = languageFeatures;

            _token = new TokenWithSpan();

            _lookahead = new TokenWithSpan();

            _fromFutureAllowed = true;

            _functions = null;

            _privatePrefix = null;



            _parsingStarted = false;

            _errorCode = 0;

        }



        public void Reset() {

            Reset(_sourceUnit, _languageFeatures);

        }



        #endregion



        #region Error Reporting



        private void ReportSyntaxError(TokenWithSpan t) {

            ReportSyntaxError(t, ErrorCodes.SyntaxError);

        }



        private void ReportSyntaxError(TokenWithSpan t, int errorCode) {

            ReportSyntaxError(t.Token, t.Span, errorCode, true);

        }



        private void ReportSyntaxError(Token t, SourceSpan span, int errorCode, bool allowIncomplete) {

            SourceLocation start = span.Start;

            SourceLocation end = span.End;



            if (allowIncomplete && (t.Kind == TokenKind.EndOfFile || (_tokenizer.IsEndOfFile && (t.Kind == TokenKind.Dedent || t.Kind == TokenKind.NLToken)))) {

                errorCode |= ErrorCodes.IncompleteStatement;

            }



            string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image);



            ReportSyntaxError(start, end, msg, errorCode);

        }



        private static string GetErrorMessage(Token t, int errorCode) {

            string msg;

            if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError) {

                msg = Resources.ExpectedIndentation;

            } else if (t.Kind != TokenKind.EndOfFile) {

                msg = Resources.UnexpectedToken;

            } else {

                msg = "unexpected EOF while parsing";

            }

            

            return msg;

        }



        private void ReportSyntaxError(string message) {

            ReportSyntaxError(_lookahead.Span.Start, _lookahead.Span.End, message);

        }



        internal void ReportSyntaxError(SourceLocation start, SourceLocation end, string message) {

            ReportSyntaxError(start, end, message, ErrorCodes.SyntaxError);

        }



        internal void ReportSyntaxError(SourceLocation start, SourceLocation end, string message, int errorCode) {

            // save the first one, the next error codes may be induced errors:

            if (_errorCode == 0) {

                _errorCode = errorCode;

            }

            _errors.Add(_sourceUnit, message, new SourceSpan(start, end), errorCode, Severity.FatalError);

        }



        #endregion        



        #region LL(1) Parsing



        private static bool IsPrivateName(string name) {

            return name.StartsWith("__") && !name.EndsWith("__");

        }



        private string FixName(string name) {

            if (_privatePrefix != null && IsPrivateName(name)) {

                name = string.Format("_{0}{1}", _privatePrefix, name);

            }



            return name;

        }



        private string ReadNameMaybeNone() {

            // peek for better error recovery

            Token t = PeekToken();

            if (t == Tokens.NoneToken) {

                NextToken();

                return "None";

            }



            NameToken n = t as NameToken;

            if (n == null) {

                ReportSyntaxError("syntax error");

                return null;

            }



            NextToken();

            return FixName(n.Name);

        }



        private string ReadName() {

            NameToken n = PeekToken() as NameToken;

            if (n == null) {

                ReportSyntaxError(_lookahead);

                return null;

            }

            NextToken();

            return FixName(n.Name);

        }



        //stmt: simple_stmt | compound_stmt

        //compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef

        private Statement ParseStmt() {

            switch (PeekToken().Kind) {

                case TokenKind.KeywordIf:

                    return ParseIfStmt();

                case TokenKind.KeywordWhile:

                    return ParseWhileStmt();

                case TokenKind.KeywordFor:

                    return ParseForStmt();

                case TokenKind.KeywordTry:

                    return ParseTryStatement();

                case TokenKind.At:

                    return ParseDecorated();

                case TokenKind.KeywordDef:

                    return ParseFuncDef();

                case TokenKind.KeywordClass:

                    return ParseClassDef();

                case TokenKind.KeywordWith:

                    return ParseWithStmt();

                default:

                    return ParseSimpleStmt();

            }

        }



        //simple_stmt: small_stmt (';' small_stmt)* [';'] Newline

        private Statement ParseSimpleStmt() {

            Statement s = ParseSmallStmt();

            if (MaybeEat(TokenKind.Semicolon)) {

                SourceLocation start = s.Start;

                List<Statement> l = new List<Statement>();

                l.Add(s);

                while (true) {

                    if (MaybeEatNewLine() || MaybeEat(TokenKind.EndOfFile)) {

                        break;

                    }



                    l.Add(ParseSmallStmt());



                    if (MaybeEat(TokenKind.EndOfFile)) {

                        // implies a new line

                        break;

                    } else if (!MaybeEat(TokenKind.Semicolon)) {

                        EatNewLine();

                        break;

                    }

                }

                Statement[] stmts = l.ToArray();



                SuiteStatement ret = new SuiteStatement(stmts);

                ret.SetLoc(start, stmts[stmts.Length - 1].End);

                return ret;

            } else if (!MaybeEat(TokenKind.EndOfFile) && !EatNewLine()) {

                // error handling, make sure we're making forward progress

                NextToken();                

            }

            return s;

        }



        /*

        small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt



        del_stmt: 'del' exprlist

        pass_stmt: 'pass'

        flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt

        break_stmt: 'break'

        continue_stmt: 'continue'

        return_stmt: 'return' [testlist]

        yield_stmt: 'yield' testlist

        */

        private Statement ParseSmallStmt() {

            switch (PeekToken().Kind) {

                case TokenKind.KeywordPrint:

                    return ParsePrintStmt();

                case TokenKind.KeywordPass:

                    return FinishSmallStmt(new EmptyStatement());

                case TokenKind.KeywordBreak:

                    if (!_inLoop) {

                        ReportSyntaxError("'break' outside loop");

                    }

                    return FinishSmallStmt(new BreakStatement());

                case TokenKind.KeywordContinue:

                    if (!_inLoop) {

                        ReportSyntaxError("'continue' not properly in loop");

                    } else if (_inFinally) {

                        ReportSyntaxError("'continue' not supported inside 'finally' clause");

                    }

                    return FinishSmallStmt(new ContinueStatement());

                case TokenKind.KeywordReturn:

                    return ParseReturnStmt();

                case TokenKind.KeywordFrom:

                    return ParseFromImportStmt();

                case TokenKind.KeywordImport:

                    return ParseImportStmt();

                case TokenKind.KeywordGlobal:

                    return ParseGlobalStmt();

                case TokenKind.KeywordRaise:

                    return ParseRaiseStmt();

                case TokenKind.KeywordAssert:

                    return ParseAssertStmt();

                case TokenKind.KeywordExec:

                    return ParseExecStmt();

                case TokenKind.KeywordDel:

                    return ParseDelStmt();

                case TokenKind.KeywordYield:

                    return ParseYieldStmt();

                default:

                    return ParseExprStmt();

            }

        }



        // del_stmt: "del" target_list

        //  for error reporting reasons we allow any expression and then report the bad

        //  delete node when it fails.  This is the reason we don't call ParseTargetList.

        private Statement ParseDelStmt() {

            NextToken();

            SourceLocation start = GetStart();

            List<Expression> l = ParseExprList();

            foreach (Expression e in l) {

                string delError = e.CheckDelete();

                if (delError != null) {

                    ReportSyntaxError(e.Span.Start, e.Span.End, delError, ErrorCodes.SyntaxError);

                }

            }



            DelStatement ret = new DelStatement(l.ToArray());

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        private Statement ParseReturnStmt() {

            if (CurrentFunction == null) {

                ReportSyntaxError(IronPython.Resources.MisplacedReturn);

            }

            NextToken();

            Expression expr = null;

            SourceLocation start = GetStart();

            if (!NeverTestToken(PeekToken())) {

                expr = ParseTestListAsExpr(true);

            }



            if (expr != null) {

                _returnWithValue = true;

                if (_isGenerator) {

                    ReportSyntaxError("'return' with argument inside generator");

                }

            }



            ReturnStatement ret = new ReturnStatement(expr);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        private Statement FinishSmallStmt(Statement stmt) {

            NextToken();

            stmt.SetLoc(GetStart(), GetEnd());

            return stmt;

        }





        private Statement ParseYieldStmt() {

            // For yield statements, continue to enforce that it's currently in a function. 

            // This gives us better syntax error reporting for yield-statements than for yield-expressions.

            FunctionDefinition current = CurrentFunction;

            if (current == null) {

                ReportSyntaxError(IronPython.Resources.MisplacedYield);

            }



            _isGenerator = true;

            if (_returnWithValue) {

                ReportSyntaxError("'return' with argument inside generator");

            }



            Eat(TokenKind.KeywordYield);



            // See Pep 342: a yield statement is now just an expression statement around a yield expression.

            Expression e = ParseYieldExpression();

            Debug.Assert(e != null); // caller already verified we have a yield.



            Statement s = new ExpressionStatement(e);

            s.SetLoc(e.Span);

            return s;

        }



        /// <summary>

        /// Peek if the next token is a 'yield' and parse a yield expression. Else return null.

        /// 

        /// Called w/ yield already eaten.

        /// </summary>

        /// <returns>A yield expression if present, else null. </returns>

        // yield_expression: "yield" [expression_list] 

        private Expression ParseYieldExpression() {

            // Mark that this function is actually a generator.

            // If we're in a generator expression, then we don't have a function yet.

            //    g=((yield i) for i in range(5))

            // In that acse, the genexp will mark IsGenerator. 

            FunctionDefinition current = CurrentFunction;

            if (current != null) {

                current.IsGenerator = true;

            }



            SourceLocation start = GetStart();



            // Parse expression list after yield. This can be:

            // 1) empty, in which case it becomes 'yield None'

            // 2) a single expression

            // 3) multiple expression, in which case it's wrapped in a tuple.

            Expression yieldResult;



            bool trailingComma;

            List<Expression> l = ParseExpressionList(out trailingComma);

            if (l.Count == 0) {

                // Check empty expression and convert to 'none'

                yieldResult = new ConstantExpression(null);

            } else if (l.Count != 1) {

                // make a tuple

                yieldResult = MakeTupleOrExpr(l, trailingComma);

            } else {

                // just take the single expression

                yieldResult = l[0];

            }



            Expression yieldExpression = new YieldExpression(yieldResult);



            yieldExpression.SetLoc(start, GetEnd());

            return yieldExpression;



        }



        private Statement FinishAssignments(Expression right) {

            List<Expression> left = new List<Expression>();



            while (MaybeEat(TokenKind.Assign)) {

                string assignError = right.CheckAssign();

                if (assignError != null) {

                    ReportSyntaxError(right.Span.Start, right.Span.End, assignError, ErrorCodes.SyntaxError | ErrorCodes.NoCaret);

                }



                left.Add(right);



                if (MaybeEat(TokenKind.KeywordYield)) {

                    right = ParseYieldExpression();

                } else {

                    bool trailingComma;

                    var exprs = ParseExpressionList(out trailingComma);

                    if (exprs.Count == 0) {

                        ReportSyntaxError(left[0].Start, left[0].End, "invalid syntax");

                    }

                    right = MakeTupleOrExpr(exprs, trailingComma);

                }

            }



            Debug.Assert(left.Count > 0);



            AssignmentStatement assign = new AssignmentStatement(left.ToArray(), right);

            assign.SetLoc(left[0].Start, right.End);

            return assign;

        }



        // expr_stmt: expression_list

        // expression_list: expression ( "," expression )* [","] 

        // assignment_stmt: (target_list "=")+ (expression_list | yield_expression) 

        // augmented_assignment_stmt ::= target augop (expression_list | yield_expression) 

        // augop: '+=' | '-=' | '*=' | '/=' | '%=' | '**=' | '>>=' | '<<=' | '&=' | '^=' | '|=' | '//='

        private Statement ParseExprStmt() {

            Expression ret = ParseTestListAsExpr(false);

            if (ret is ErrorExpression) {

                NextToken();

            }



            if (PeekToken(TokenKind.Assign)) {

                return FinishAssignments(ret);

            } else {

                PythonOperator op = GetAssignOperator(PeekToken());

                if (op != PythonOperator.None) {

                    NextToken();

                    Expression rhs;



                    if (MaybeEat(TokenKind.KeywordYield)) {

                        rhs = ParseYieldExpression();

                    } else {

                        rhs = ParseTestListAsExpr(false);

                    }



                    string assignError = ret.CheckAugmentedAssign();

                    if (assignError != null) {

                        ReportSyntaxError(assignError);

                    }



                    AugmentedAssignStatement aug = new AugmentedAssignStatement(op, ret, rhs);

                    aug.SetLoc(ret.Start, GetEnd());

                    return aug;

                } else {

                    Statement stmt = new ExpressionStatement(ret);

                    stmt.SetLoc(ret.Span);

                    return stmt;

                }

            }

        }



        private PythonOperator GetAssignOperator(Token t) {

            switch (t.Kind) {

                case TokenKind.AddEqual: return PythonOperator.Add;

                case TokenKind.SubtractEqual: return PythonOperator.Subtract;

                case TokenKind.MultiplyEqual: return PythonOperator.Multiply;

                case TokenKind.DivideEqual: return TrueDivision ? PythonOperator.TrueDivide : PythonOperator.Divide;

                case TokenKind.ModEqual: return PythonOperator.Mod;

                case TokenKind.BitwiseAndEqual: return PythonOperator.BitwiseAnd;

                case TokenKind.BitwiseOrEqual: return PythonOperator.BitwiseOr;

                case TokenKind.ExclusiveOrEqual: return PythonOperator.Xor;

                case TokenKind.LeftShiftEqual: return PythonOperator.LeftShift;

                case TokenKind.RightShiftEqual: return PythonOperator.RightShift;

                case TokenKind.PowerEqual: return PythonOperator.Power;

                case TokenKind.FloorDivideEqual: return PythonOperator.FloorDivide;

                default: return PythonOperator.None;

            }

        }





        private PythonOperator GetBinaryOperator(OperatorToken token) {

            switch (token.Kind) {

                case TokenKind.Add: return PythonOperator.Add;

                case TokenKind.Subtract: return PythonOperator.Subtract;

                case TokenKind.Multiply: return PythonOperator.Multiply;

                case TokenKind.Divide: return TrueDivision ? PythonOperator.TrueDivide : PythonOperator.Divide;

                case TokenKind.Mod: return PythonOperator.Mod;

                case TokenKind.BitwiseAnd: return PythonOperator.BitwiseAnd;

                case TokenKind.BitwiseOr: return PythonOperator.BitwiseOr;

                case TokenKind.ExclusiveOr: return PythonOperator.Xor;

                case TokenKind.LeftShift: return PythonOperator.LeftShift;

                case TokenKind.RightShift: return PythonOperator.RightShift;

                case TokenKind.Power: return PythonOperator.Power;

                case TokenKind.FloorDivide: return PythonOperator.FloorDivide;

                default:

                    string message = String.Format(

                        System.Globalization.CultureInfo.InvariantCulture,

                        Resources.UnexpectedToken,

                        token.Kind);

                    Debug.Assert(false, message);

                    throw new ArgumentException(message);

            }

        }





        // import_stmt: 'import' module ['as' name"] (',' module ['as' name])*        

        // name: identifier

        private ImportStatement ParseImportStmt() {

            Eat(TokenKind.KeywordImport);

            SourceLocation start = GetStart();



            List<ModuleName> l = new List<ModuleName>();

            List<string> las = new List<string>();

            l.Add(ParseModuleName());

            las.Add(MaybeParseAsName());

            while (MaybeEat(TokenKind.Comma)) {

                l.Add(ParseModuleName());

                las.Add(MaybeParseAsName());

            }

            ModuleName[] names = l.ToArray();

            var asNames = las.ToArray();



            ImportStatement ret = new ImportStatement(names, asNames, AbsoluteImports);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        // module: (identifier '.')* identifier

        private ModuleName ParseModuleName() {

            SourceLocation start = GetStart();

            ModuleName ret = new ModuleName(ReadNames());

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        // relative_module: "."* module | "."+

        private ModuleName ParseRelativeModuleName() {

            SourceLocation start = GetStart();



            int dotCount = 0;

            while (MaybeEat(TokenKind.Dot)) {

                dotCount++;

            }



            string[] names = ArrayUtils.EmptyStrings;

            if (PeekToken() is NameToken) {

                names = ReadNames();

            }



            ModuleName ret;

            if (dotCount > 0) {

                ret = new RelativeModuleName(names, dotCount);

            } else {

                if (names.Length == 0) {

                    ReportSyntaxError(_lookahead.Span.Start, _lookahead.Span.End, "invalid syntax");

                }

                ret = new ModuleName(names);

            }



            ret.SetLoc(start, GetEnd());

            return ret;

        }



        private string[] ReadNames() {

            List<string> l = new List<string>();

            l.Add(ReadName());

            while (MaybeEat(TokenKind.Dot)) {

                l.Add(ReadName());

            }

            return l.ToArray();

        }





        // 'from' relative_module 'import' identifier ['as' name] (',' identifier ['as' name]) *

        // 'from' relative_module 'import' '(' identifier ['as' name] (',' identifier ['as' name])* [','] ')'        

        // 'from' module 'import' "*"                                        

        private FromImportStatement ParseFromImportStmt() {

            Eat(TokenKind.KeywordFrom);

            SourceLocation start = GetStart();

            ModuleName dname = ParseRelativeModuleName();



            Eat(TokenKind.KeywordImport);



            bool ateParen = MaybeEat(TokenKind.LeftParenthesis);



            string[] names;

            string[] asNames;

            bool fromFuture = false;



            if (MaybeEat(TokenKind.Multiply)) {

                names = (string[])FromImportStatement.Star;

                asNames = null;

            } else {

                List<string> l = new List<string>();

                List<string> las = new List<string>();



                if (MaybeEat(TokenKind.LeftParenthesis)) {

                    ParseAsNameList(l, las);

                    Eat(TokenKind.RightParenthesis);

                } else {

                    ParseAsNameList(l, las);

                }

                names = l.ToArray();

                asNames = las.ToArray();

            }



            // Process from __future__ statement



            if (dname.Names.Count == 1 && dname.Names[0] == "__future__") {

                if (!_fromFutureAllowed) {

                    ReportSyntaxError(IronPython.Resources.MisplacedFuture);

                }

                if (names == FromImportStatement.Star) {

                    ReportSyntaxError(IronPython.Resources.NoFutureStar);

                }

                fromFuture = true;

                foreach (string name in names) {

                    if (name == "division") {

                        _languageFeatures |= ModuleOptions.TrueDivision;

                    } else if (name == "with_statement") {

                        _languageFeatures |= ModuleOptions.WithStatement;

                    } else if (name == "absolute_import") {

                        _languageFeatures |= ModuleOptions.AbsoluteImports;

                    } else if (name == "print_function") {

                        _languageFeatures |= ModuleOptions.PrintFunction;

                        _tokenizer.PrintFunction = true;

                    } else if (name == "unicode_literals") {

                        _tokenizer.UnicodeLiterals = true;

                        _languageFeatures |= ModuleOptions.UnicodeLiterals;

                    } else if (name == "nested_scopes") {

                    } else if (name == "generators") {

                    } else {

                        string strName = name;

                        fromFuture = false;



                        if (strName != "braces") {

                            ReportSyntaxError(IronPython.Resources.UnknownFutureFeature + strName);

                        } else {

                            // match CPython error message

                            ReportSyntaxError(IronPython.Resources.NotAChance);

                        }

                    }

                }

            }



            if (ateParen) {

                Eat(TokenKind.RightParenthesis);

            }



            FromImportStatement ret = new FromImportStatement(dname, (string[])names, asNames, fromFuture, AbsoluteImports);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        // import_as_name (',' import_as_name)*

        private void ParseAsNameList(List<string> l, List<string> las) {

            l.Add(ReadName());

            las.Add(MaybeParseAsName());

            while (MaybeEat(TokenKind.Comma)) {

                if (PeekToken(TokenKind.RightParenthesis)) return;  // the list is allowed to end with a ,

                l.Add(ReadName());

                las.Add(MaybeParseAsName());

            }

        }



        //import_as_name: NAME [NAME NAME]

        //dotted_as_name: dotted_name [NAME NAME]

        private string MaybeParseAsName() {

            if (MaybeEat(TokenKind.KeywordAs)) {

                return ReadName();

            }

            return null;

        }



        //exec_stmt: 'exec' expr ['in' expression [',' expression]]

        private ExecStatement ParseExecStmt() {

            Eat(TokenKind.KeywordExec);

            SourceLocation start = GetStart();

            Expression code, locals = null, globals = null;

            code = ParseExpr();

            if (MaybeEat(TokenKind.KeywordIn)) {

                globals = ParseExpression();

                if (MaybeEat(TokenKind.Comma)) {

                    locals = ParseExpression();

                }

            }

            ExecStatement ret = new ExecStatement(code, locals, globals);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        //global_stmt: 'global' NAME (',' NAME)*

        private GlobalStatement ParseGlobalStmt() {

            Eat(TokenKind.KeywordGlobal);

            SourceLocation start = GetStart();

            List<string> l = new List<string>();

            l.Add(ReadName());

            while (MaybeEat(TokenKind.Comma)) {

                l.Add(ReadName());

            }

            string[] names = l.ToArray();

            GlobalStatement ret = new GlobalStatement(names);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        //raise_stmt: 'raise' [expression [',' expression [',' expression]]]

        private RaiseStatement ParseRaiseStmt() {

            Eat(TokenKind.KeywordRaise);

            SourceLocation start = GetStart();

            Expression type = null, _value = null, traceback = null;



            if (!NeverTestToken(PeekToken())) {

                type = ParseExpression();

                if (MaybeEat(TokenKind.Comma)) {

                    _value = ParseExpression();

                    if (MaybeEat(TokenKind.Comma)) {

                        traceback = ParseExpression();

                    }

                }

            }

            RaiseStatement ret = new RaiseStatement(type, _value, traceback);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        //assert_stmt: 'assert' expression [',' expression]

        private AssertStatement ParseAssertStmt() {

            Eat(TokenKind.KeywordAssert);

            SourceLocation start = GetStart();

            Expression expr = ParseExpression();

            Expression message = null;

            if (MaybeEat(TokenKind.Comma)) {

                message = ParseExpression();

            }

            AssertStatement ret = new AssertStatement(expr, message);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        //print_stmt: 'print' ( [ expression (',' expression)* [','] ] | '>>' expression [ (',' expression)+ [','] ] )

        private PrintStatement ParsePrintStmt() {

            Eat(TokenKind.KeywordPrint);

            SourceLocation start = GetStart();

            Expression dest = null;

            PrintStatement ret;



            bool needNonEmptyTestList = false;

            if (MaybeEat(TokenKind.RightShift)) {

                dest = ParseExpression();

                if (MaybeEat(TokenKind.Comma)) {

                    needNonEmptyTestList = true;

                } else {

                    ret = new PrintStatement(dest, new Expression[0], false);

                    ret.SetLoc(start, GetEnd());

                    return ret;

                }

            }



            bool trailingComma;

            List<Expression> l = ParseExpressionList(out trailingComma);

            if (needNonEmptyTestList && l.Count == 0) {

                ReportSyntaxError(_lookahead);

            }

            Expression[] exprs = l.ToArray();

            ret = new PrintStatement(dest, exprs, trailingComma);

            ret.SetLoc(start, GetEnd());

            return ret;

        }



        private string SetPrivatePrefix(string name) {

            string oldPrefix = _privatePrefix;



            _privatePrefix = GetPrivatePrefix(name);



            return oldPrefix;

        }



        internal static string GetPrivatePrefix(string name) {

            // Remove any leading underscores before saving the prefix

            if (name != null) {

                for (int i = 0; i < name.Length; i++) {

                    if (name[i] != '_') {

                        return name.Substring(i);

                    }

                }

            }

            // Name consists of '_'s only, no private prefix mapping

            return null;

        }



        private ErrorExpression Error() {

            var res = new ErrorExpression();

            res.SetLoc(GetStart(), GetEnd());

            return res;

        }



        private ExpressionStatement ErrorStmt() {

            return new ExpressionStatement(Error());

        }



        //classdef: 'class' NAME ['(' testlist ')'] ':' suite

        private ClassDefinition ParseClassDef() {

            Eat(TokenKind.KeywordClass);



            SourceLocation start = GetStart();

            string name = ReadName();

            if (name == null) {

                // no name, assume there's no class.

                return new ClassDefinition(null, new Expression[0], ErrorStmt());

            }



            Expression[] bases = new Expression[0];

            if (MaybeEat(TokenKind.LeftParenthesis)) {

                List<Expression> l = ParseTestList();



                if (l.Count == 1 && l[0] is ErrorExpression) {

                    // error handling, classes is incomplete.

                    return new ClassDefinition(name, new Expression[0], ErrorStmt());

                }

                bases = l.ToArray();

                Eat(TokenKind.RightParenthesis);

            }

            SourceLocation mid = GetEnd();



            // Save private prefix

            string savedPrefix = SetPrivatePrefix(name);



            // Parse the class body

            Statement body = ParseClassOrFuncBody();



            // Restore the private prefix

            _privatePrefix = savedPrefix;



            ClassDefinition ret = new ClassDefinition(name, bases, body);

            ret.Header = mid;

            ret.SetLoc(start, GetEnd());

            return ret;

        }





        //  decorators ::=

        //      decorator+

        //  decorator ::=

        //      "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE

        private List<Expression> ParseDecorators() {

            List<Expression> decorators = new List<Expression>();



            while (MaybeEat(TokenKind.At)) {

                SourceLocation start = GetStart();

                Expression decorator = new NameExpression(ReadName());

                decorator.SetLoc(start, GetEnd());

                while (MaybeEat(TokenKind.Dot)) {

                    string name = ReadNameMaybeNone();

                    decorator = new MemberExpression(decorator, name);

                    decorator.SetLoc(GetStart(), GetEnd());

                }

                decorator.SetLoc(start, GetEnd());



                if (MaybeEat(TokenKind.LeftParenthesis)) {

                    _sink.StartParameters(GetSpan());

                    Arg[] args = FinishArgumentList(null);

                    decorator = FinishCallExpr(decorator, args);

                }

                decorator.SetLoc(start, GetEnd());

                EatNewLine();



                decorators.Add(decorator);

            }



            return decorators;

        }



        // funcdef: [decorators] 'def' NAME parameters ':' suite

        // 2.6: 

        //  decorated: decorators (classdef | funcdef)

        // this gets called with "@" look-ahead

        private Statement ParseDecorated() {

            List<Expression> decorators = ParseDecorators();



            Statement res;



            if (PeekToken() == Tokens.KeywordDefToken) {

                FunctionDefinition fnc = ParseFuncDef();

                fnc.Decorators = decorators.ToArray();

                res = fnc;

            } else if (PeekToken() == Tokens.KeywordClassToken) {

                ClassDefinition cls = ParseClassDef();

                cls.Decorators = decorators.ToArray();

                res = cls;

            } else {

                res = new EmptyStatement();

                ReportSyntaxError(_lookahead);

            }

            

            return res;

        }



        // funcdef: [decorators] 'def' NAME parameters ':' suite

        // parameters: '(' [varargslist] ')'

        // this gets called with "def" as the look-ahead

        private FunctionDefinition ParseFuncDef() {

            Eat(TokenKind.KeywordDef);

            SourceLocation start = GetStart();

            string name = ReadName();



            Eat(TokenKind.LeftParenthesis);



            SourceLocation lStart = GetStart(), lEnd = GetEnd();

            int grouping = _tokenizer.GroupingLevel;



            Parameter[] parameters = ParseVarArgsList(TokenKind.RightParenthesis);

            FunctionDefinition ret;

            if (parameters == null) {

                // error in parameters

                ret = new FunctionDefinition(name, new Parameter[0]);

                ret.SetLoc(start, lEnd);

                return ret;

            }

            

            SourceLocation rStart = GetStart(), rEnd = GetEnd();



            ret = new FunctionDefinition(name, parameters);

            PushFunction(ret);





            Statement body = ParseClassOrFuncBody();

            FunctionDefinition ret2 = PopFunction();

            System.Diagnostics.Debug.Assert(ret == ret2);



            ret.Body = body;

            ret.Header = rEnd;



            _sink.MatchPair(new SourceSpan(lStart, lEnd), new SourceSpan(rStart, rEnd), grouping);



            ret.SetLoc(start, body.End);



            return ret;

        }



        private Parameter ParseParameterName(Dictionary<string, object> names, ParameterKind kind) {

            string name = ReadName();

            if (name != null) {

                CheckUniqueParameter(names, name);

            } else {

                return null;

            }

            Parameter parameter = new Parameter(name, kind);

            parameter.SetLoc(GetStart(), GetEnd());

            return parameter;

        }



        private void CheckUniqueParameter(Dictionary<string, object> names, string name) {

            if (names.ContainsKey(name)) {

                ReportSyntaxError(String.Format(

                    System.Globalization.CultureInfo.InvariantCulture,

                    Resources.DuplicateArgumentInFuncDef,

                    name));

            }

            names[name] = null;

        }



        //varargslist: (fpdef ['=' expression ] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | fpdef ['=' expression] (',' fpdef ['=' expression])* [',']

        //fpdef: NAME | '(' fplist ')'

        //fplist: fpdef (',' fpdef)* [',']

        private Parameter[] ParseVarArgsList(TokenKind terminator) {

            // parameters not doing * or ** today

            List<Parameter> pl = new List<Parameter>();

            Dictionary<string, object> names = new Dictionary<string, object>(StringComparer.Ordinal);

            bool needDefault = false;

            for (int position = 0; ; position++) {

                if (MaybeEat(terminator)) break;



                Parameter parameter;



                if (MaybeEat(TokenKind.Multiply)) {

                    parameter = ParseParameterName(names, ParameterKind.List);

                    if (parameter == null) {

                        // no parameter name, syntax error

                        return null;

                    }

                    pl.Add(parameter);

                    if (MaybeEat(TokenKind.Comma)) {

                        Eat(TokenKind.Power);

                        parameter = ParseParameterName(names, ParameterKind.Dictionary);

                        if (parameter == null) {

                            return null;

                        }

                        pl.Add(parameter);

                    }

                    Eat(terminator);

                    break;

                } else if (MaybeEat(TokenKind.Power)) {

                    parameter = ParseParameterName(names, ParameterKind.Dictionary);

                    if (parameter == null) {

                        // no parameter name, syntax error

                        return null;

                    }

                    pl.Add(parameter);

                    Eat(terminator);

                    break;

                }



                //

                //  Parsing defparameter:

                //

                //  defparameter ::=

                //      parameter ["=" expression]



                if ((parameter = ParseParameter(position, names)) != null) {

                    pl.Add(parameter);

                    if (MaybeEat(TokenKind.Assign)) {

                        needDefault = true;

                        parameter.DefaultValue = ParseExpression();

                    } else if (needDefault) {

                        ReportSyntaxError(IronPython.Resources.DefaultRequired);

                    }

                } else {

                    // no parameter due to syntax error

                    return null;

                }



                if (!MaybeEat(TokenKind.Comma)) {

                    Eat(terminator);

                    break;

                }

            }



            return pl.ToArray();

        }



        //  parameter ::=

        //      identifier | "(" sublist ")"

        private Parameter ParseParameter(int position, Dictionary<string, object> names) {

            Token t = PeekToken();

            Parameter parameter = null;



            switch (t.Ki…

Large files files are truncated, but you can click here to view the full file