PageRenderTime 49ms CodeModel.GetById 15ms RepoModel.GetById 1ms app.codeStats 0ms

/Backend/AST/Parser.cs

https://bitbucket.org/AdamMil/boaold
C# | 1279 lines | 1062 code | 92 blank | 125 comment | 303 complexity | a7a755773c011f40894e67e84bbe6b1b MD5 | raw file
Possible License(s): GPL-2.0
  1. /*
  2. Boa is the reference implementation for a language similar to Python,
  3. also called Boa. This implementation is both interpreted and compiled,
  4. targeting the Microsoft .NET Framework.
  5. http://www.adammil.net/
  6. Copyright (C) 2004-2005 Adam Milazzo
  7. This program is free software; you can redistribute it and/or
  8. modify it under the terms of the GNU General Public License
  9. as published by the Free Software Foundation; either version 2
  10. of the License, or (at your option) any later version.
  11. This program is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. GNU General Public License for more details.
  15. You should have received a copy of the GNU General Public License
  16. along with this program; if not, write to the Free Software
  17. Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18. */
  19. using System;
  20. using System.Collections;
  21. using System.Collections.Specialized;
  22. using System.IO;
  23. using System.Text;
  24. using Boa.Runtime;
  25. // TODO: add switch?
  26. // TODO: add <=> operator
  27. // TODO: implement sets
  28. // TODO: try to make precedence match python's where it makes sense
  29. // TODO: support unicode string parsing
  30. // FIXME: make this parse: (lambda: print)
  31. // TODO: detect assignment to constants in the parser itself
  32. // TODO: make string parsing closer to python's if possible
  33. // TODO: add proper parsing of octal numbers
  34. // TODO: add support for parsing numbers in any base (eg 3#1212 is 1212 using base 3)
  35. namespace Boa.AST
  36. {
  37. #region Tokens
  38. enum Token
  39. { None,
  40. // punctuation
  41. Period, Comma, BackQuote, LParen, RParen, LBrace, RBrace, LBracket, RBracket, Question, Colon,
  42. Semicolon, Percent, Plus, Minus, Asterisk, Slash,
  43. // punctuation operators
  44. Power, FloorDivide, LeftShift, RightShift,
  45. BitAnd, BitOr, BitNot, BitXor, LogAnd, LogOr, LogNot,
  46. // keywords
  47. Def, Print, Return, While, If, Elif, Else, Pass, Break, Continue, Global, Import, From, For, In, Not,
  48. Lambda, Try, Except, Finally, Raise, Class, Assert, Is, Del, Yield, Lock, Using,
  49. // abstract
  50. Identifier, Literal, Assign, Compare, Call, Member, Index, Slice, Hash, List, Tuple, Suite,
  51. Module, Assembly, EOL, EOF
  52. }
  53. #endregion
  54. #region Parser
  55. public sealed class Parser
  56. { public Parser(Stream data) : this("<unknown>", data) { }
  57. public Parser(string source, Stream data) : this(source, new StreamReader(data), false) { }
  58. public Parser(string source, TextReader data) : this(source, data, false) { }
  59. public Parser(string source, TextReader data, bool autoclose)
  60. { sourceFile = source; this.data = data.ReadToEnd();
  61. if(autoclose) data.Close();
  62. NextToken();
  63. }
  64. public Parser(string source, string data)
  65. { sourceFile=source; this.data=data;
  66. NextToken();
  67. }
  68. static Parser()
  69. { stringTokens = new Hashtable();
  70. Token[] tokens =
  71. { Token.Def, Token.Print, Token.Return, Token.While, Token.Import, Token.From,
  72. Token.For, Token.If, Token.Elif, Token.Else, Token.Pass, Token.Break, Token.Continue, Token.Global, Token.In,
  73. Token.Lambda, Token.Try, Token.Except, Token.Finally, Token.Raise, Token.Class, Token.Assert, Token.Is,
  74. Token.Del, Token.Yield, Token.Not, Token.Lock, Token.Using,
  75. };
  76. foreach(Token token in tokens) stringTokens.Add(Enum.GetName(typeof(Token), token).ToLower(), token);
  77. stringTokens.Add("and", Token.LogAnd);
  78. stringTokens.Add("or", Token.LogOr);
  79. }
  80. public static Parser FromFile(string filename) { return new Parser(filename, new StreamReader(filename), true); }
  81. public static Parser FromStream(Stream stream) { return new Parser("<stream>", new StreamReader(stream)); }
  82. public static Parser FromString(string text) { return new Parser("<string>", text); }
  83. public Statement Parse()
  84. { ArrayList stmts = new ArrayList();
  85. while(true)
  86. { if(TryEat(Token.EOL)) continue;
  87. if(TryEat(Token.EOF)) break;
  88. int line = this.line, column = this.column;
  89. Statement stmt = ParseStatement();
  90. stmt.SetLocation(sourceFile, line, column);
  91. stmts.Add(stmt);
  92. }
  93. return stmts.Count==0 ? new PassStatement() : (Statement)new Suite((Statement[])stmts.ToArray(typeof(Statement)));
  94. }
  95. // expression := <ternary> | <lambda> (',' <expression>)?
  96. public Expression ParseExpression()
  97. { if(token==Token.Lambda) return ParseLambda();
  98. Expression expr = ParseTernary();
  99. if(bareTuples && token==Token.Comma)
  100. { ArrayList exprs = new ArrayList();
  101. exprs.Add(expr);
  102. bareTuples = false;
  103. while(TryEat(Token.Comma) && token!=Token.EOL && token!=Token.Assign) exprs.Add(ParseExpression());
  104. bareTuples = true;
  105. expr = AP(new TupleExpression((Expression[])exprs.ToArray(typeof(Expression))));
  106. }
  107. return expr;
  108. }
  109. public BoaFunction ParseFunction() { return ((DefStatement)ParseDef()).Function; }
  110. // statement := <stmt_line> | <compound_stmt>
  111. // compount_stmt := <if_stmt> | <while_stmt> | <for_stmt> | <def_stmt> | <try_stmt> | <global_stmt> |
  112. // <import_stmt> | <class_stmt> | <label_stmt> | <lock_stmt> | <using_stmt>
  113. // label_stmt := <identitier> <suite>
  114. public Statement ParseStatement()
  115. { switch(token)
  116. { case Token.If: return ParseIf();
  117. case Token.While: return ParseWhile();
  118. case Token.For: return ParseFor();
  119. case Token.Def: return ParseDef();
  120. case Token.Try: return ParseTry();
  121. case Token.Global: return ParseGlobal();
  122. case Token.Class: return ParseClass();
  123. case Token.Import: case Token.From: return ParseImport();
  124. case Token.Lock: case Token.Using: return ParseUsingBlock(token);
  125. default:
  126. if(token==Token.Identifier)
  127. { string label = (string)value;
  128. if(PeekToken()==Token.Colon)
  129. { NextToken();
  130. Statement st = ParseSuite();
  131. if(!(st is Suite)) st = new Suite(new Statement[] { st });
  132. ((Suite)st).Name = label;
  133. return st;
  134. }
  135. }
  136. return ParseStmtLine();
  137. }
  138. }
  139. bool InLoop { get { return loopDepth>0; } }
  140. Expression AP(Expression e) { e.SetLocation(sourceFile, line, column); return e; }
  141. Node AP(Node n) { n.SetLocation(sourceFile, line, column); return n; }
  142. Statement AP(Statement s) { s.SetLocation(sourceFile, line, column); return s; }
  143. #region GetEscapeChar
  144. /*
  145. \newline Ignored
  146. \\ Backslash
  147. \" Double quotation mark
  148. \' Single quotation mark
  149. \n Newline
  150. \t Tab
  151. \r Carriage return
  152. \b Backspace
  153. \e Escape
  154. \a Bell
  155. \f Form feed
  156. \v Vertical tab
  157. \xHH Up to 2 hex digits -> byte value
  158. \uHHHH Up to 4 hex digits -> 16-bit unicode value
  159. \cC Control code (eg, \cC is ctrl-c)
  160. \OOO Up to 3 octal digits -> byte value
  161. */
  162. char GetEscapeChar()
  163. { char c = ReadChar();
  164. if(char.IsDigit(c))
  165. { if(c>'7') SyntaxError("invalid octal digit");
  166. int num = (c-'0');
  167. for(int i=1; i<3; i++)
  168. { c = ReadChar();
  169. if(!char.IsDigit(c) || c>'7') { lastChar=c; break; }
  170. num = (num<<3) | (c-'0');
  171. }
  172. return (char)num;
  173. }
  174. else switch(c)
  175. { case '\"': return '\"';
  176. case '\'': return '\'';
  177. case 'n': return '\n';
  178. case 't': return '\t';
  179. case 'r': return '\r';
  180. case 'b': return '\b';
  181. case 'e': return (char)27;
  182. case 'a': return '\a';
  183. case 'f': return '\f';
  184. case 'v': return '\v';
  185. case '\\': return '\\';
  186. case 'x': case 'u':
  187. { int num = 0;
  188. for(int i=0,limit=(c=='x'?2:4); i<limit; i++)
  189. { c = ReadChar();
  190. if(char.IsDigit(c)) num = (num<<4) | (c-'0');
  191. else if((c<'A' || c>'F') && (c<'a' || c>'f'))
  192. { if(i==0) SyntaxError("expected hex digit");
  193. lastChar = c;
  194. break;
  195. }
  196. else num = (num<<4) | (char.ToUpper(c)-'A'+10);
  197. }
  198. return (char)num;
  199. }
  200. case 'c':
  201. c = ReadChar();
  202. if(!char.IsLetter(c)) SyntaxError("expected letter");
  203. return (char)(char.ToUpper(c)-64);
  204. case '\n': return '\0';
  205. default: SyntaxError(string.Format("unknown escape character '{0}'", c)); return c;
  206. }
  207. }
  208. #endregion
  209. void Eat(Token type) { if(token!=type) Unexpected(token, type); NextToken(); }
  210. void Expect(Token type) { if(token!=type) Unexpected(token); }
  211. Token NextToken()
  212. { if(nextToken!=Token.None)
  213. { token = nextToken;
  214. value = nextValue;
  215. nextToken = Token.None;
  216. }
  217. else token = ReadToken();
  218. return token;
  219. }
  220. // argument_list := <argument> (',' <argument>)*
  221. // argument := ('*' | '**')? <expression> | <identifier> '=' <expression>
  222. Argument[] ParseArguments()
  223. { bool obt=bareTuples, owe=wantEOL;
  224. bareTuples=wantEOL=false;
  225. try
  226. { Eat(Token.LParen);
  227. if(token==Token.RParen) return new Argument[0];
  228. ArrayList args = new ArrayList();
  229. do
  230. { if(TryEat(Token.Asterisk)) args.Add(new Argument(ParseExpression(), ArgType.List));
  231. else if(TryEat(Token.Power)) args.Add(new Argument(ParseExpression(), ArgType.Dict));
  232. else if(token!=Token.Identifier) args.Add(new Argument(ParseExpression()));
  233. else
  234. { Expression e = ParseExpression();
  235. if(TryEat(Token.Assign))
  236. { if(!(e is NameExpression)) Unexpected(Token.Assign);
  237. args.Add(new Argument(((NameExpression)e).Name.String, ParseExpression()));
  238. }
  239. else args.Add(new Argument(e));
  240. }
  241. } while(TryEat(Token.Comma));
  242. ListDictionary ld = new ListDictionary();
  243. foreach(Argument a in args)
  244. if(a.Name!=null)
  245. { if(ld.Contains(a.Name)) SyntaxError("duplicate keyword argument '{0}'", a.Name);
  246. else ld[a.Name] = null;
  247. }
  248. return (Argument[])args.ToArray(typeof(Argument));
  249. }
  250. finally { bareTuples=obt; wantEOL=owe; }
  251. }
  252. // bitwise := <shift> (<bitwise_op> <shift>)*
  253. // bitwise_op := '&' | '|' | '^'
  254. Expression ParseBitwise()
  255. { Expression expr = ParseShift();
  256. while(true)
  257. { BinaryOperator op;
  258. switch(token)
  259. { case Token.BitAnd: op = BinaryOperator.BitwiseAnd; break;
  260. case Token.BitOr: op = BinaryOperator.BitwiseOr; break;
  261. case Token.BitXor: op = BinaryOperator.BitwiseXor; break;
  262. default: return expr;
  263. }
  264. NextToken();
  265. expr = new BinaryOpExpression(op, expr, ParseShift());
  266. }
  267. }
  268. // cim_expr := <primary> ('(' <argument_list> ')' | '[' <index> ']' | '.' <identifier>)*
  269. Expression ParseCIM()
  270. { Expression expr = ParsePrimary();
  271. while(true)
  272. { if(token==Token.LParen)
  273. { expr = AP(new CallExpression(expr, ParseArguments()));
  274. Eat(Token.RParen);
  275. }
  276. else if(TryEat(Token.LBracket))
  277. { Expression start = token==Token.Colon ? null : ParseExpression();
  278. if(TryEat(Token.Colon))
  279. { Expression stop = token==Token.Colon || token==Token.RBracket ? null : ParseExpression();
  280. Expression step = TryEat(Token.Colon) ? ParseExpression() : null;
  281. start = AP(new SliceExpression(start, stop, step));
  282. }
  283. Eat(Token.RBracket);
  284. expr = AP(new IndexExpression(expr, start));
  285. }
  286. else if(TryEat(Token.Period)) expr = AP(new AttrExpression(expr, ParseIdentifier()));
  287. else return expr;
  288. }
  289. }
  290. // class_stmt := 'class' <identifier> <inheritance>? <suite>
  291. // inheritance := '(' <expr_list> ')'
  292. Statement ParseClass()
  293. { Eat(Token.Class);
  294. string name = ParseIdentifier();
  295. Expression[] bases=null;
  296. if(TryEat(Token.LParen))
  297. { if(TryEat(Token.RParen)) bases = new Expression[0];
  298. else
  299. { ArrayList inherit = new ArrayList();
  300. bool obt = bareTuples;
  301. bareTuples = false;
  302. do inherit.Add(ParseExpression()); while(TryEat(Token.Comma));
  303. Eat(Token.RParen);
  304. bases = (Expression[])inherit.ToArray(typeof(Expression));
  305. bareTuples = obt;
  306. }
  307. }
  308. else bases = new Expression[0];
  309. return AP(new ClassStatement(name, bases, ParseSuite()));
  310. }
  311. // compare := <isin> (<compare_op> <isin>)*
  312. // compare_op := '==' | '!=' | '<' | '>' | '<=' | '>=' | '<>' | 'is' | 'is not'
  313. Expression ParseCompare()
  314. { Expression expr = ParseIsIn();
  315. ArrayList comps = null;
  316. while(true)
  317. { if(token==Token.Compare || token==Token.Is)
  318. { if(comps==null)
  319. { comps = new ArrayList();
  320. comps.Add(expr);
  321. }
  322. if(token==Token.Compare)
  323. { comps.Add(value);
  324. NextToken();
  325. }
  326. else // token==Token.Is
  327. { if(comps==null) comps = new ArrayList();
  328. bool not = NextToken()==Token.Not;
  329. comps.Add(not ? BinaryOperator.NotIdentical : (BinaryOperator)BinaryOperator.Identical);
  330. if(not) NextToken();
  331. }
  332. comps.Add(ParseIsIn());
  333. }
  334. else break;
  335. }
  336. if(comps!=null)
  337. { if(comps.Count==3)
  338. expr = new BinaryOpExpression((BinaryOperator)comps[1], (Expression)comps[0], (Expression)comps[2]);
  339. else
  340. { Expression[] exprs = new Expression[(comps.Count+1)/2];
  341. BinaryOperator[] ops = new BinaryOperator[comps.Count/2];
  342. exprs[0] = (Expression)comps[0];
  343. for(int i=1,j=0; i<comps.Count; )
  344. { ops[j] = (BinaryOperator)comps[i++];
  345. exprs[++j] = (Expression)comps[i++];
  346. }
  347. expr = new CompareExpression(exprs, ops);
  348. }
  349. }
  350. return expr;
  351. }
  352. // def_stmt := 'def' <identifier> '(' <param_list> ')' ':' <suite>
  353. Statement ParseDef()
  354. { Eat(Token.Def);
  355. string name = ParseIdentifier();
  356. Eat(Token.LParen);
  357. Parameter[] parms = ParseParamList(Token.RParen);
  358. Eat(Token.RParen);
  359. return AP(new DefStatement(name, parms, ParseSuite()));
  360. }
  361. // module := <identifier> ('.' <identifier>)*
  362. string ParseDotted()
  363. { string ret = ParseIdentifier();
  364. while(TryEat(Token.Period)) ret += '.' + ParseIdentifier();
  365. return ret;
  366. }
  367. // expr_stmt := (<lvalue> '=')* <expression>
  368. // assignable := <name> | <index> | <slice>
  369. // lvalue := <assignable> | <tuple of <assignable>>
  370. Statement ParseExprStmt()
  371. { Expression lhs = ParseExpression();
  372. if(token==Token.Assign)
  373. { ArrayList list = new ArrayList();
  374. AssignStatement ass = (AssignStatement)AP(new AssignStatement());
  375. ass.Op = (BinaryOperator)value;
  376. while(TryEat(Token.Assign))
  377. { if(ass.Op!=null)
  378. { if(list.Count>0) SyntaxError("can't chain in-place assignment");
  379. if(!(lhs is NameExpression || lhs is AttrExpression || lhs is IndexExpression))
  380. SyntaxError("can't do in-place assignment with {0}", lhs.GetType());
  381. }
  382. else if(!(lhs is NameExpression || lhs is AttrExpression || lhs is TupleExpression || lhs is IndexExpression))
  383. SyntaxError("can't assign to {0}", lhs.GetType());
  384. list.Add(lhs);
  385. lhs = ParseExpression();
  386. }
  387. ass.LHS = (Expression[])list.ToArray(typeof(Expression));
  388. ass.RHS = lhs;
  389. return ass;
  390. }
  391. else return new ExpressionStatement(lhs);
  392. }
  393. // factor := <power> (<factor_op> <power>)*
  394. // factor_op := '*' | '/' | '%' | '//'
  395. Expression ParseFactor()
  396. { Expression expr = ParsePower();
  397. while(true)
  398. { BinaryOperator op;
  399. switch(token)
  400. { case Token.Asterisk: op = BinaryOperator.Multiply; break;
  401. case Token.Slash: op = BinaryOperator.Divide; break;
  402. case Token.Percent: op = BinaryOperator.Modulus; break;
  403. case Token.FloorDivide: op = BinaryOperator.FloorDivide; break;
  404. default: return expr;
  405. }
  406. NextToken();
  407. expr = new BinaryOpExpression(op, expr, ParsePower());
  408. }
  409. }
  410. // for_stmt := 'for' <namelist> 'in' <expression> <suite>
  411. Statement ParseFor()
  412. { int indent=this.indent;
  413. Eat(Token.For);
  414. Name[] names = ParseNameList();
  415. Eat(Token.In);
  416. Expression loopExp = ParseExpression();
  417. loopDepth++;
  418. Statement body = ParseSuite(), elze=this.indent==indent && TryEat(Token.Else) ? ParseSuite() : null;
  419. loopDepth--;
  420. return AP(new ForStatement(names, loopExp, body, elze));
  421. }
  422. // global_stmt := 'global' <namelist> EOL
  423. Statement ParseGlobal()
  424. { Eat(Token.Global);
  425. ArrayList names = new ArrayList();
  426. do names.Add(ParseIdentifier()); while(TryEat(Token.Comma));
  427. Eat(Token.EOL);
  428. return AP(new GlobalStatement((string[])names.ToArray(typeof(string))));
  429. }
  430. string ParseIdentifier()
  431. { Expect(Token.Identifier);
  432. string ret = (string)value;
  433. NextToken();
  434. return ret;
  435. }
  436. // if_stmt := 'if' <expression> <suite> ('elif' <expression> <suite>)* ('else' <suite>)?
  437. Statement ParseIf()
  438. { if(token!=Token.If && token!=Token.Elif) Unexpected(token);
  439. int indent=this.indent;
  440. NextToken();
  441. Expression test = ParseExpression();
  442. Statement body = ParseSuite(), elze=null;
  443. if(this.indent==indent)
  444. { if(token==Token.Elif) elze = ParseIf();
  445. else if(TryEat(Token.Else)) elze = ParseSuite();
  446. }
  447. return AP(new IfStatement(test, body, elze));
  448. }
  449. // import_stmt := 'import' <import_package> (',' <import_package>)* EOL |
  450. // 'from' <dotted> 'import' <import_ident> (',' <import_ident>)* EOL |
  451. // 'from' <dotted> 'import' '*' EOL
  452. // import_package := <dotted> ('as' <identifier>)?
  453. // import_ident := <identifier> ('as' <identifier>)?
  454. Statement ParseImport()
  455. { Statement stmt;
  456. if(TryEat(Token.From))
  457. { string module = ParseDotted();
  458. Eat(Token.Import);
  459. if(TryEat(Token.Asterisk)) stmt = AP(new ImportFromStatement(module, new ImportName("*")));
  460. else
  461. { Expect(Token.Identifier);
  462. ArrayList list = new ArrayList();
  463. do
  464. { string ident = ParseIdentifier();
  465. if(token==Token.Identifier && (string)value=="as")
  466. { NextToken();
  467. list.Add(new ImportName(ident, ParseIdentifier()));
  468. }
  469. else list.Add(new ImportName(ident));
  470. } while(token!=Token.EOL && TryEat(Token.Comma));
  471. stmt = AP(new ImportFromStatement(module, (ImportName[])list.ToArray(typeof(ImportName))));
  472. }
  473. }
  474. else
  475. { Eat(Token.Import);
  476. ArrayList list = new ArrayList();
  477. do
  478. { string module = ParseDotted();
  479. if(token==Token.Identifier && (string)value=="as")
  480. { NextToken();
  481. list.Add(new ImportName(module, ParseIdentifier()));
  482. }
  483. else list.Add(new ImportName(module));
  484. } while(token!=Token.EOL && TryEat(Token.Comma));
  485. stmt = AP(new ImportStatement((ImportName[])list.ToArray(typeof(ImportName))));
  486. }
  487. Eat(Token.EOL);
  488. return stmt;
  489. }
  490. // isin := <bitwise> ('not'? 'in' <bitwise>)*
  491. Expression ParseIsIn()
  492. { Expression expr = ParseBitwise();
  493. while(true)
  494. { if(TryEat(Token.In)) expr = AP(new InExpression(expr, ParseBitwise(), false));
  495. else if(TryEat(Token.Not))
  496. { Eat(Token.In);
  497. expr = AP(new InExpression(expr, ParseBitwise(), true));
  498. }
  499. else return expr;
  500. }
  501. }
  502. // lambda := 'lambda' <namelist> ':' <lambda_body>
  503. // lambda_body := <simple_stmt> (';' <simple_stmt>)* <lambda_end>
  504. // lambda_end := EOL | ',' | ')' | ']' | '}' | 'for'
  505. Expression ParseLambda()
  506. { Eat(Token.Lambda);
  507. Parameter[] parms = ParseParamList(Token.Colon);
  508. Eat(Token.Colon);
  509. ArrayList list = new ArrayList();
  510. do
  511. { switch(token)
  512. { case Token.EOL: case Token.Comma: case Token.RParen: case Token.RBrace: case Token.RBracket: case Token.For:
  513. goto done;
  514. case Token.Return:
  515. switch(NextToken())
  516. { case Token.EOL: case Token.Comma: case Token.RParen: case Token.RBrace: case Token.RBracket:
  517. case Token.For:
  518. list.Add(AP(new ReturnStatement())); break;
  519. default: list.Add(AP(new ReturnStatement(ParseExpression()))); break;
  520. }
  521. break;
  522. default: list.Add(ParseSimpleStmt()); break;
  523. }
  524. } while(TryEat(Token.Semicolon));
  525. done:
  526. if(list.Count==0) Unexpected(token);
  527. return AP(new LambdaExpression(parms, new Suite((Statement[])list.ToArray(typeof(Statement)))));
  528. }
  529. // list_comprehension := <expression> ('for' <namelist> 'in' <expression> ('if' <expression>)?)+
  530. Expression ParseListComprehension(Expression expr)
  531. { ArrayList list = new ArrayList();
  532. do
  533. { Eat(Token.For);
  534. Name[] names = ParseNameList();
  535. Eat(Token.In);
  536. list.Add(new ListCompFor(names, ParseExpression(), TryEat(Token.If) ? ParseExpression() : null));
  537. } while(token==Token.For);
  538. return AP(new ListCompExpression(expr, (ListCompFor[])list.ToArray(typeof(ListCompFor))));
  539. }
  540. // logand := <lognot> ('&&' <lognot>)*
  541. Expression ParseLogAnd()
  542. { Expression expr = ParseLogNot();
  543. while(TryEat(Token.LogAnd)) expr = AP(new AndExpression(expr, ParseLogNot()));
  544. return expr;
  545. }
  546. // lognot := 'not' <lognot> | <compare>
  547. Expression ParseLogNot()
  548. { return TryEat(Token.Not) ? new UnaryExpression(ParseLogNot(), UnaryOperator.LogicalNot) : ParseCompare();
  549. }
  550. // logor := <logand> ('||' <logand>)*
  551. Expression ParseLogOr()
  552. { Expression expr = ParseLogAnd();
  553. while(TryEat(Token.LogOr)) expr = AP(new OrExpression(expr, ParseLogAnd()));
  554. return expr;
  555. }
  556. // primary := LITERAL | <ident> | '(' <expression> ')' | '[' <array_list> ']' | '{' <hash_list> '}' |
  557. // '[' <list_comprehension> ']' | <tuple of <expression>> | '`' <expression> '`'
  558. // tuple of T := '(' (<T> ',')+ <T>? ')'
  559. Expression ParsePrimary()
  560. { Expression expr;
  561. bool obt=bareTuples, owe=wantEOL;
  562. switch(token)
  563. { case Token.Literal:
  564. if(value is string)
  565. { ConstantExpression e = (ConstantExpression)AP(new ConstantExpression(null));
  566. string s = (string)value;
  567. while(NextToken()==Token.Literal && value is string) s += (string)value;
  568. if(token==Token.Literal) SyntaxError("unexpected literal '{0}' in string concatenation");
  569. e.Value = s;
  570. return e;
  571. }
  572. else expr = AP(new ConstantExpression(value));
  573. break;
  574. case Token.Identifier: expr = AP(new NameExpression(new Name((string)value))); break;
  575. case Token.LParen:
  576. bareTuples=wantEOL=false;
  577. NextToken();
  578. if(token==Token.RParen) expr = AP(new TupleExpression());
  579. else
  580. { expr = ParseExpression();
  581. if(token==Token.Comma)
  582. { NextToken();
  583. ArrayList list = new ArrayList();
  584. list.Add(expr);
  585. while(token!=Token.RParen)
  586. { bareTuples = false;
  587. list.Add(ParseExpression());
  588. if(!TryEat(Token.Comma)) break;
  589. }
  590. expr = AP(new TupleExpression((Expression[])list.ToArray(typeof(Expression))));
  591. }
  592. else if(token==Token.For) // hijack ParseListComprehension()
  593. { ListCompExpression lc = (ListCompExpression)ParseListComprehension(expr);
  594. expr = new GeneratorExpression(lc.Item, lc.Fors);
  595. expr.SetLocation(lc);
  596. }
  597. else expr = AP(new ParenExpression(expr));
  598. }
  599. Expect(Token.RParen);
  600. break;
  601. case Token.LBracket:
  602. bareTuples=wantEOL=false;
  603. NextToken();
  604. if(token==Token.RBracket) expr = AP(new ListExpression());
  605. else
  606. { Expression fe = ParseExpression();
  607. if(token==Token.For) expr = ParseListComprehension(fe);
  608. else
  609. { ArrayList list = new ArrayList();
  610. list.Add(fe);
  611. TryEat(Token.Comma);
  612. while(token!=Token.RBracket)
  613. { bareTuples = false;
  614. list.Add(ParseExpression());
  615. if(!TryEat(Token.Comma)) break;
  616. }
  617. expr = AP(new ListExpression((Expression[])list.ToArray(typeof(Expression))));
  618. }
  619. }
  620. Expect(Token.RBracket);
  621. break;
  622. case Token.LBrace:
  623. bareTuples=wantEOL=false;
  624. NextToken();
  625. if(token==Token.RBrace) expr = AP(new HashExpression());
  626. else
  627. { ArrayList list = new ArrayList();
  628. while(token!=Token.RBrace)
  629. { Expression e = ParseExpression();
  630. Eat(Token.Colon);
  631. list.Add(new DictionaryEntry(e, ParseExpression()));
  632. if(!TryEat(Token.Comma)) break;
  633. }
  634. expr = AP(new HashExpression((DictionaryEntry[])list.ToArray(typeof(DictionaryEntry))));
  635. }
  636. Expect(Token.RBrace);
  637. break;
  638. case Token.BackQuote:
  639. NextToken();
  640. expr = ParseExpression();
  641. Expect(Token.BackQuote);
  642. expr = AP(new ReprExpression(expr));
  643. break;
  644. default: Unexpected(token); return null;
  645. }
  646. bareTuples=obt; wantEOL=owe;
  647. NextToken();
  648. return expr;
  649. }
  650. // print_stmt := 'print' (<expression> (',' <expression>)* ','?)? |
  651. // 'print' '>>' <expression> (',' <expression> (',' <expression>)* ','?)?
  652. Statement ParsePrintStmt()
  653. { Eat(Token.Print);
  654. if(token==Token.EOL || token==Token.Semicolon) return AP(new PrintStatement());
  655. bool comma, old=bareTuples;
  656. bareTuples = false;
  657. Expression file = TryEat(Token.RightShift) ? ParseExpression() : null;
  658. if(file!=null)
  659. { if(token==Token.EOL || token==Token.Semicolon)
  660. { bareTuples=old;
  661. return AP(new PrintStatement(file));
  662. }
  663. Eat(Token.Comma);
  664. }
  665. ArrayList stmts = new ArrayList();
  666. do
  667. { comma=false;
  668. stmts.Add(ParseExpression());
  669. if(TryEat(Token.Comma)) comma=true;
  670. } while(token!=Token.EOL && token!=Token.Semicolon);
  671. bareTuples = old;
  672. return AP(new PrintStatement(file, (Expression[])stmts.ToArray(typeof(Expression)), !comma));
  673. }
  674. // namelist := <identifier> (',' <identifier>)*
  675. Name[] ParseNameList()
  676. { ArrayList list = new ArrayList();
  677. do list.Add(new Name(ParseIdentifier())); while(TryEat(Token.Comma));
  678. return (Name[])list.ToArray(typeof(Name));
  679. }
  680. // param_list := <required_params>? <pcomma> <optional_params>? <pcomma> ('*' <identifier>)? <pcomma>
  681. // ('**' <identifier>)?
  682. // required_params := <identifier> (',' <identifier>)
  683. // optional_params := <optional_param> (',' <optional_param>)*
  684. // optional_param := <identifier> '=' <expression>
  685. // pcomma := ','? (comma required to separate argument/parameter groups)
  686. Parameter[] ParseParamList(Token end)
  687. { if(token==end) return new Parameter[0];
  688. ArrayList parms = new ArrayList();
  689. string ident;
  690. bool obt=bareTuples, owe=wantEOL;
  691. bareTuples=wantEOL=false;
  692. while(true) // required identifiers
  693. { if(TryEat(Token.Asterisk)) goto list;
  694. if(token==Token.Power) goto dict;
  695. ident = ParseIdentifier();
  696. if(TryEat(Token.Assign)) break;
  697. parms.Add(new Parameter(ident));
  698. if(token==end) goto done;
  699. Eat(Token.Comma);
  700. }
  701. while(true) // positional parameters
  702. { parms.Add(new Parameter(ident, ParseExpression()));
  703. if(token==end) goto done;
  704. Eat(Token.Comma);
  705. if(TryEat(Token.Asterisk)) break;
  706. if(token==Token.Power) goto dict;
  707. ident = ParseIdentifier();
  708. Eat(Token.Assign);
  709. }
  710. list: if(token==Token.Identifier) parms.Add(new Parameter(ParseIdentifier(), ParamType.List));
  711. if(token==end) goto done;
  712. Eat(Token.Comma);
  713. dict: Eat(Token.Power);
  714. parms.Add(new Parameter(ParseIdentifier(), ParamType.Dict));
  715. done:
  716. ListDictionary ld = new ListDictionary();
  717. foreach(Parameter p in parms)
  718. if(ld.Contains(p.Name.String)) SyntaxError("duplicate parameter name '{0}'", p.Name.String);
  719. else ld[p.Name.String] = null;
  720. bareTuples=obt; wantEOL=owe;
  721. return (Parameter[])parms.ToArray(typeof(Parameter));
  722. }
  723. // power := <unary> ('**' <unary>)*
  724. Expression ParsePower()
  725. { Expression expr = ParseUnary();
  726. while(TryEat(Token.Power)) expr = new BinaryOpExpression(BinaryOperator.Power, expr, ParseUnary());
  727. return expr;
  728. }
  729. // shift := <term> (('<<' | '>>') <term>)*
  730. Expression ParseShift()
  731. { Expression expr = ParseTerm();
  732. while(true)
  733. { BinaryOperator op;
  734. switch(token)
  735. { case Token.LeftShift: op = BinaryOperator.LeftShift; break;
  736. case Token.RightShift: op = BinaryOperator.RightShift; break;
  737. default: return expr;
  738. }
  739. NextToken();
  740. expr = new BinaryOpExpression(op, expr, ParseTerm());
  741. }
  742. }
  743. // simple_stmt := <expr_stmt> | <print_stmt> | <break_stmt> | <continue_stmt> | <pass_stmt> | <return_stmt>
  744. // <assert_stmt> | <del_stmt> | <yield_stmt> | <goto_stmt>
  745. // break_stmt := 'break' EOL
  746. // continue_stmt := 'continue' EOL
  747. // pass_stmt := 'pass' EOL
  748. // raise_stmt := 'raise' <expression>?
  749. // return_stmt := 'return' <expression>?
  750. // assert_stmt := 'assert' <expression>
  751. // yield_stmt := 'yield' <expression>
  752. // del_stmt := 'del' <lvalue> (',' <lvalue>)*
  753. // goto_stmt := 'goto' <identifier>
  754. Statement ParseSimpleStmt()
  755. { switch(token)
  756. { case Token.Print: return ParsePrintStmt();
  757. case Token.Break:
  758. { BreakStatement bs=null;
  759. if(NextToken()==Token.Identifier)
  760. { bs = new BreakStatement((string)value);
  761. NextToken();
  762. }
  763. else if(!InLoop) SyntaxError("'break' encountered outside loop");
  764. else bs = new BreakStatement();
  765. return AP(bs);
  766. }
  767. case Token.Continue:
  768. { ContinueStatement cs=null;
  769. if(NextToken()==Token.Identifier)
  770. { cs = new ContinueStatement((string)value);
  771. NextToken();
  772. }
  773. else if(!InLoop) SyntaxError("'continue' encountered outside loop");
  774. else cs = new ContinueStatement();
  775. return AP(cs);
  776. }
  777. case Token.Pass: NextToken(); return AP(new PassStatement());
  778. case Token.Return:
  779. NextToken();
  780. return AP(token==Token.EOL || token==Token.Semicolon ? new ReturnStatement()
  781. : new ReturnStatement(ParseExpression()));
  782. case Token.Raise:
  783. NextToken();
  784. return AP(token==Token.EOL || token==Token.Semicolon ? new RaiseStatement()
  785. : new RaiseStatement(ParseExpression()));
  786. case Token.Assert: NextToken(); return AP(new AssertStatement(ParseExpression()));
  787. case Token.Yield: NextToken(); return AP(new YieldStatement(ParseExpression()));
  788. case Token.Del:
  789. { NextToken();
  790. ArrayList list = new ArrayList();
  791. do
  792. { Expression e = ParseExpression();
  793. if(!(e is NameExpression || e is AttrExpression || e is TupleExpression || e is IndexExpression))
  794. SyntaxError("can't delete {0}", e.GetType());
  795. list.Add(e);
  796. } while(TryEat(Token.Comma));
  797. return AP(new DelStatement((Expression[])list.ToArray(typeof(Expression))));
  798. }
  799. default: return ParseExprStmt();
  800. }
  801. }
  802. // stmt_line := <simple_stmt> (';' <simple_stmt>)* (NEWLINE | EOF)
  803. Statement ParseStmtLine()
  804. { Statement stmt = ParseSimpleStmt();
  805. if(token==Token.Semicolon)
  806. { ArrayList stmts = new ArrayList();
  807. stmts.Add(stmt);
  808. while(TryEat(Token.Semicolon)) stmts.Add(ParseSimpleStmt());
  809. stmt = new Suite((Statement[])stmts.ToArray(typeof(Statement)));
  810. }
  811. Eat(Token.EOL);
  812. return stmt;
  813. }
  814. // suite := ':' stmt_line | ':'? NEWLINE INDENT <statement>+ UNINDENT
  815. Statement ParseSuite()
  816. { if(TryEat(Token.Colon) && token!=Token.EOL) return ParseStmtLine();
  817. int indent=this.indent;
  818. Eat(Token.EOL);
  819. if(this.indent<=indent) SyntaxError("expected indent");
  820. ArrayList stmts = new ArrayList();
  821. while(this.indent>indent)
  822. { if(TryEat(Token.EOL)) continue;
  823. stmts.Add(ParseStatement());
  824. }
  825. return new Suite((Statement[])stmts.ToArray(typeof(Statement)));
  826. }
  827. // term := <factor> (('+' | '-') <factor>)*
  828. Expression ParseTerm()
  829. { Expression expr = ParseFactor();
  830. while(true)
  831. { BinaryOperator op;
  832. switch(token)
  833. { case Token.Plus: op = BinaryOperator.Add; break;
  834. case Token.Minus: op = BinaryOperator.Subtract; break;
  835. default: return expr;
  836. }
  837. NextToken();
  838. expr = new BinaryOpExpression(op, expr, ParseFactor());
  839. }
  840. }
  841. // ternary := <logor> ('?' <expression> ':' <expression>)
  842. Expression ParseTernary()
  843. { Expression expr = ParseLogOr();
  844. if(TryEat(Token.Question))
  845. { Expression it = ParseExpression();
  846. Eat(Token.Colon);
  847. expr = AP(new TernaryExpression(expr, it, ParseExpression()));
  848. }
  849. return expr;
  850. }
  851. // try := 'try' <suite> NEWLINE ('except' <expression>? ',' <ident> <suite>)*
  852. // ('except' <suite>)? ('else' <suite>)? ('finally' <suite>)?
  853. Statement ParseTry()
  854. { int indent=this.indent;
  855. Eat(Token.Try);
  856. Statement body = ParseSuite(), elze=null, final=null;
  857. ArrayList list = new ArrayList();
  858. while(indent==this.indent && TryEat(Token.Except))
  859. { bool obt=bareTuples;
  860. bareTuples=false;
  861. Expression type = token==Token.Comma || token==Token.Colon || token==Token.EOL ? null : ParseExpression();
  862. bareTuples=true;
  863. NameExpression name = TryEat(Token.Comma) ? (NameExpression)AP(new NameExpression(ParseIdentifier())) : null;
  864. list.Add(AP((Node)new ExceptClause(type, name, ParseSuite())));
  865. if(type==null) break;
  866. }
  867. if(indent==this.indent && TryEat(Token.Else)) elze = ParseSuite();
  868. if(indent==this.indent && TryEat(Token.Finally)) final = ParseSuite();
  869. if(list.Count==0 && elze==null && final==null) SyntaxError("expecting 'except', 'else', or 'finally'");
  870. return AP(new TryStatement(body, (ExceptClause[])list.ToArray(typeof(ExceptClause)), elze, final));
  871. }
  872. // unary := <unary_op> <unary>
  873. // unary_op := '!' | '~' | '-' | '+'
  874. Expression ParseUnary()
  875. { UnaryOperator op=null;
  876. switch(token)
  877. { case Token.LogNot: op = UnaryOperator.LogicalNot; break;
  878. case Token.Minus: op = UnaryOperator.UnaryMinus; break;
  879. case Token.BitNot: op = UnaryOperator.BitwiseNot; break;
  880. case Token.Plus: NextToken(); return ParseUnary();
  881. }
  882. if(op!=null)
  883. { NextToken();
  884. return AP(new UnaryExpression(ParseUnary(), op));
  885. }
  886. return ParseCIM();
  887. }
  888. // lock_stmt ::= 'lock' <expression> <suite>
  889. // using_stmt ::= 'using' <expression> <suite>
  890. Statement ParseUsingBlock(Token token)
  891. { Eat(token);
  892. Expression expr = ParseExpression();
  893. Statement body = ParseSuite();
  894. return AP(token==Token.Lock ? new LockStatement(expr, body) : (Statement)new UsingStatement(expr, body));
  895. }
  896. // while_stmt := 'while' <expression> <suite>
  897. Statement ParseWhile()
  898. { Eat(Token.While);
  899. Expression expr = ParseExpression();
  900. loopDepth++;
  901. Statement body = ParseSuite(), elze=this.indent==indent && TryEat(Token.Else) ? ParseSuite() : null;
  902. loopDepth--;
  903. return AP(new WhileStatement(expr, body, elze));
  904. }
  905. Token PeekToken()
  906. { if(nextToken!=Token.None) return nextToken;
  907. return nextToken = ReadToken(ref nextValue);
  908. }
  909. char ReadChar()
  910. { char c;
  911. if(lastChar!=0) { c=lastChar; lastChar='\0'; return c; }
  912. else if(pos>=data.Length) { indent=-1; return '\0'; }
  913. c = data[pos++]; column++;
  914. if(c=='\n') { line++; column=1; }
  915. else if(c=='\r')
  916. { if(pos<data.Length && data[pos]=='\n') pos++;
  917. c='\n'; line++; column=1;
  918. }
  919. else if(c==0) c = ' ';
  920. return c;
  921. }
  922. #region ReadToken
  923. Token ReadToken() { return ReadToken(ref value); }
  924. Token ReadToken(ref object value)
  925. { char c;
  926. while(true)
  927. { if(token==Token.EOL)
  928. { c=ReadChar();
  929. if(wantEOL)
  930. { indent=0;
  931. while(c!=0 && (char.IsWhiteSpace(c) || c=='#'))
  932. { if(c=='\n') indent=0;
  933. else if(c=='#') { do c = ReadChar(); while(c!='\n' && c!=0); indent=0; }
  934. else indent++;
  935. c=ReadChar();
  936. }
  937. }
  938. else while(c!=0 && char.IsWhiteSpace(c)) c=ReadChar();
  939. }
  940. else do c=ReadChar(); while(c!='\n' && c!=0 && char.IsWhiteSpace(c));
  941. if(char.IsDigit(c) || c=='.')
  942. { if(c=='.')
  943. { lastChar = ReadChar();
  944. if(!char.IsDigit(lastChar)) return Token.Period;
  945. }
  946. string s=string.Empty;
  947. bool period=false, hex=false;
  948. while(true)
  949. { if(c=='.')
  950. { if(hex) break;
  951. if(period) SyntaxError("invalid number");
  952. period = true;
  953. }
  954. else if(!char.IsDigit(c) && (!hex || (c<'a' || c>'f') && (c<'A' || c>'F')))
  955. { if(!hex && (c=='x' || c=='X') && s=="0") { s=string.Empty; hex=true; goto nextchar; }
  956. break;
  957. }
  958. s += c;
  959. nextchar: c = ReadChar();
  960. }
  961. try
  962. { if(char.ToUpper(c)=='J')
  963. { if(hex) SyntaxError("'J' modifier cannot be used with hex numbers");
  964. value = new Complex(0, double.Parse(s));
  965. }
  966. else if(char.ToUpper(c)=='E')
  967. { if(hex) SyntaxError("'E' modifier cannot be used with hex numbers");
  968. double num=double.Parse(s), exp;
  969. bool neg=false;
  970. s = string.Empty;
  971. c = ReadChar();
  972. if(c=='-') { neg=true; c=ReadChar(); }
  973. else if(c=='+') c=ReadChar();
  974. if(!char.IsNumber(c)) SyntaxError("Expected number in scientific notation.");
  975. do
  976. { s += c;
  977. c = ReadChar();
  978. } while(char.IsNumber(c));
  979. lastChar = c;
  980. exp = double.Parse(s);
  981. value = num * Math.Pow(10, neg ? -exp : exp);
  982. }
  983. else if(period)
  984. { if(hex) SyntaxError("invalid number or attribute reference");
  985. if(char.ToUpper(c)=='L') throw new NotImplementedException("decimal type");
  986. else { lastChar=c; value = double.Parse(s); }
  987. }
  988. else
  989. { if(char.ToUpper(c)!='L') lastChar=c;
  990. try { value = hex ? Convert.ToInt32(s, 16) : int.Parse(s); }
  991. catch(OverflowException)
  992. { try { value = hex ? Convert.ToInt64(s, 16) : long.Parse(s); }
  993. catch(OverflowException) { value = Integer.Parse(s, hex ? 16 : 10); }
  994. }
  995. }
  996. return Token.Literal;
  997. }
  998. catch(FormatException) { SyntaxError("invalid number"); }
  999. }
  1000. else if(c=='_' || char.IsLetter(c))
  1001. { StringBuilder sb = new StringBuilder();
  1002. if(c=='r')
  1003. { char temp=c; c=ReadChar();
  1004. if(c=='\"' || c=='\'')
  1005. { char delim = c;
  1006. while((c=ReadChar())!=0 && c!=delim) sb.Append(c);
  1007. if(c==0) SyntaxError("unterminated string literal");
  1008. value = sb.ToString();
  1009. return Token.Literal;
  1010. }
  1011. else sb.Append(temp);
  1012. }
  1013. while(c=='_' || char.IsLetterOrDigit(c)) { sb.Append(c); c = ReadChar(); }
  1014. lastChar = c;
  1015. string s = sb.ToString();
  1016. if(s=="null" || s=="None") { value=null; return Token.Literal; }
  1017. if(s=="true" || s=="True") { value=true; return Token.Literal; }
  1018. if(s=="false" || s=="False") { value=false; return Token.Literal; }
  1019. value = stringTokens[s];
  1020. if(value!=null) return (Token)value;
  1021. value = s;
  1022. return Token.Identifier;
  1023. }
  1024. else switch(c)
  1025. { case '\n': newline: if(wantEOL) return Token.EOL; else { token=Token.EOL; break; }
  1026. case '\"': case '\'':
  1027. { StringBuilder sb = new StringBuilder();
  1028. char delim = c;
  1029. bool triple = false;
  1030. c = ReadChar();
  1031. if(c==delim)
  1032. { c = ReadChar();
  1033. if(c==delim) triple = true;
  1034. else { lastChar=c; value=string.Empty; return Token.Literal; }
  1035. }
  1036. else if(c=='\\') { char e = GetEscapeChar(); if(e!=0) sb.Append(e); }
  1037. else sb.Append(c);
  1038. while(true)
  1039. { c = ReadChar();
  1040. if(c=='\\') { char e = GetEscapeChar(); if(e!=0) sb.Append(e); }
  1041. else if(c==delim)
  1042. { if(!triple) break;
  1043. if((c=ReadChar())==delim)
  1044. { if((c=ReadChar())==delim) break;
  1045. else
  1046. { sb.Append(delim, 2);
  1047. if(c=='\\') { char e = GetEscapeChar(); if(e!=0) sb.Append(e); }
  1048. else sb.Append(c);
  1049. }
  1050. }
  1051. else
  1052. { sb.Append(delim);
  1053. if(c=='\\') { char e = GetEscapeChar(); if(e!=0) sb.Append(e); }
  1054. else sb.Append(c);
  1055. }
  1056. }
  1057. else if(c==0) SyntaxError("unterminated string literal");
  1058. else sb.Append(c);
  1059. }
  1060. value = sb.ToString();
  1061. return Token.Literal;
  1062. }
  1063. case '<':
  1064. c = ReadChar();
  1065. if(c=='<') return Token.LeftShift;
  1066. if(c=='=') value=BinaryOperator.LessEqual;
  1067. else if(c=='>') value = BinaryOperator.NotEqual;
  1068. else { lastChar = c; value = BinaryOperator.Less; }
  1069. return Token.Compare;
  1070. case '>':
  1071. c = ReadChar();
  1072. if(c=='>') return Token.RightShift;
  1073. if(c=='=') value=BinaryOperator.MoreEqual;
  1074. else { lastChar = c; value = BinaryOperator.More; }
  1075. return Token.Compare;
  1076. case '=':
  1077. c = ReadChar();
  1078. if(c=='=') { value=BinaryOperator.Equal; return Token.Compare; }
  1079. else { lastChar=c; value=null; return Token.Assign; }
  1080. case '!':
  1081. c = ReadChar();
  1082. if(c=='=') { value=BinaryOperator.NotEqual; return Token.Compare; }
  1083. else { lastChar = c; return Token.LogNot; }
  1084. case '&':
  1085. c = ReadChar();
  1086. if(c=='&')
  1087. { c = ReadChar();
  1088. if(c=='=') { value=BinaryOperator.LogicalAnd; return Token.Assign; }
  1089. lastChar = c; return Token.LogAnd;
  1090. }
  1091. if(c=='=') { value=BinaryOperator.BitwiseAnd; return Token.Assign; }
  1092. lastChar = c; return Token.BitAnd;
  1093. case '|':
  1094. c = ReadChar();
  1095. if(c=='|')
  1096. { c = ReadChar();
  1097. if(c=='=') { value=BinaryOperator.LogicalOr; return Token.Assign; }
  1098. lastChar = c; return Token.LogOr;
  1099. }
  1100. if(c=='=') { value=BinaryOperator.BitwiseOr; return Token.Assign; }
  1101. lastChar = c; return Token.BitOr;
  1102. case '^':
  1103. c = ReadChar();
  1104. if(c=='=') { value=BinaryOperator.BitwiseXor; return Token.Assign; }
  1105. lastChar = c; return Token.BitXor;
  1106. case '+':
  1107. c = ReadChar();
  1108. if(c=='=') { value=BinaryOperator.Add; return Token.Assign; }
  1109. lastChar = c; return Token.Plus;
  1110. case '-':
  1111. c = ReadChar();
  1112. if(c=='=') { value=BinaryOperator.Subtract; return Token.Assign; }
  1113. lastChar = c; return Token.Minus;
  1114. case '*':
  1115. c = ReadChar();
  1116. if(c=='=') { value=BinaryOperator.Multiply; return Token.Assign; }
  1117. if(c=='*')
  1118. { c = ReadChar();
  1119. if(c=='=') { value=BinaryOperator.Power; return Token.Assign; }
  1120. lastChar = c; return Token.Power;
  1121. }
  1122. lastChar = c; return Token.Asterisk;
  1123. case '/':
  1124. c = ReadChar();
  1125. if(c=='/')
  1126. { c = ReadChar();
  1127. if(c=='=') { value=BinaryOperator.FloorDivide; return Token.Assign; }
  1128. lastChar = c; return Token.FloorDivide;
  1129. }
  1130. if(c=='=') { value=BinaryOperator.Divide; return Token.Assign; }
  1131. if(c=='*')
  1132. { do c = ReadChar(); while(c!=0 && (c!='*' || (c=ReadChar())!='/'));
  1133. break;
  1134. }
  1135. lastChar = c; return Token.Slash;
  1136. case '%':
  1137. c = ReadChar();
  1138. if(c=='=') { value=BinaryOperator.Modulus; return Token.Assign; }
  1139. lastChar = c; return Token.Percent;
  1140. case '~': return Token.BitNot;
  1141. case ':': return Token.Colon;
  1142. case '`': return Token.BackQuote;
  1143. case ',': return Token.Comma;
  1144. case '(': return Token.LParen;
  1145. case ')': return Token.RParen;
  1146. case '[': return Token.LBracket;
  1147. case ']': return Token.RBracket;
  1148. case '{': return Token.LBrace;
  1149. case '}': return Token.RBrace;
  1150. case '?': return Token.Question;
  1151. case ';': return Token.Semicolon;
  1152. case '#':
  1153. do c = ReadChar(); while(c!='\n' && c!=0);
  1154. goto newline;
  1155. case '\\':
  1156. c = ReadChar();
  1157. if(c=='\n') break;
  1158. goto default;
  1159. case '\0': if(wantEOL) { nextToken=Token.EOF; return Token.EOL; } else return Token.EOF;
  1160. default: SyntaxError(string.Format("unexpected character '{0}' (0x{1:X})", c, (int)c)); break;
  1161. }
  1162. }
  1163. }
  1164. #endregion
  1165. void SyntaxError(string format, params object[] args)
  1166. { throw Ops.SyntaxError("{0}({1},{2}): {3}", sourceFile, line, column, string.Format(format, args));
  1167. }
  1168. bool TryEat(Token type)
  1169. { if(token==type) { NextToken(); return true; }
  1170. return false;
  1171. }
  1172. void Unexpected(Token token) { SyntaxError("unexpected token {0}", token, sourceFile, line, column); }
  1173. void Unexpected(Token got, Token expect)
  1174. { SyntaxError("unexpected token {0} (expecting {1})", got, expect, sourceFile, line, column);
  1175. }
  1176. string sourceFile, data;
  1177. Token token=Token.EOL, nextToken=Token.None;
  1178. object value, nextValue;
  1179. int line=1, column=1, pos, indent, loopDepth;
  1180. char lastChar;
  1181. bool bareTuples=true, wantEOL=true;
  1182. static Hashtable stringTokens;
  1183. }
  1184. #endregion
  1185. } // namespace Boa.AST