PageRenderTime 179ms CodeModel.GetById 80ms app.highlight 78ms RepoModel.GetById 13ms app.codeStats 0ms

/csep/src/com/aptana/editor/coffee/parsing/lexer/CoffeeRewriter.java

https://bitbucket.org/adamschmideg/coffeescript-eclipse
Java | 871 lines | 779 code | 59 blank | 33 comment | 251 complexity | 38bf69a5a4d41adf0498a9607ba012f5 MD5 | raw file
  1/**
  2 * Aptana Studio
  3 * Copyright (c) 2005-2011 by Appcelerator, Inc. All Rights Reserved.
  4 * Licensed under the terms of the GNU Public License (GPL) v3 (with exceptions).
  5 * Please see the license.html included with this distribution for details.
  6 * Any modifications to this file must keep this entire header intact.
  7 */
  8package com.aptana.editor.coffee.parsing.lexer;
  9
 10import java.text.MessageFormat;
 11import java.util.ArrayList;
 12import java.util.HashMap;
 13import java.util.HashSet;
 14import java.util.List;
 15import java.util.Map;
 16import java.util.Set;
 17import java.util.Stack;
 18
 19import com.aptana.editor.coffee.parsing.Terminals;
 20
 21@SuppressWarnings("nls")
 22public class CoffeeRewriter
 23{
 24
 25	private static Map<Short, Short> BALANCED_PAIRS = new HashMap<Short, Short>();
 26	static
 27	{
 28		BALANCED_PAIRS.put(Terminals.LPAREN, Terminals.RPAREN);
 29		BALANCED_PAIRS.put(Terminals.LBRACKET, Terminals.RBRACKET);
 30		BALANCED_PAIRS.put(Terminals.LCURLY, Terminals.RCURLY);
 31		BALANCED_PAIRS.put(Terminals.INDENT, Terminals.OUTDENT);
 32		BALANCED_PAIRS.put(Terminals.CALL_START, Terminals.CALL_END);
 33		BALANCED_PAIRS.put(Terminals.PARAM_START, Terminals.PARAM_END);
 34		BALANCED_PAIRS.put(Terminals.INDEX_START, Terminals.INDEX_END);
 35	}
 36
 37	private static Map<Short, Short> INVERSES = new HashMap<Short, Short>();
 38	private static Set<Short> EXPRESSION_START = new HashSet<Short>();
 39	private static Set<Short> EXPRESSION_END = new HashSet<Short>();
 40	static
 41	{
 42		for (Map.Entry<Short, Short> entry : BALANCED_PAIRS.entrySet())
 43		{
 44			Short left = entry.getKey();
 45			Short right = entry.getValue();
 46			INVERSES.put(right, left);
 47			INVERSES.put(left, right);
 48			EXPRESSION_START.add(left);
 49			EXPRESSION_END.add(right);
 50		}
 51	}
 52
 53	private static Set<Short> EXPRESSION_CLOSE = new HashSet<Short>();
 54	static
 55	{
 56		EXPRESSION_CLOSE.add(Terminals.CATCH);
 57		EXPRESSION_CLOSE.add(Terminals.WHEN);
 58		EXPRESSION_CLOSE.add(Terminals.ELSE);
 59		EXPRESSION_CLOSE.add(Terminals.FINALLY);
 60		EXPRESSION_CLOSE.addAll(EXPRESSION_END);
 61	}
 62
 63	private static Set<Short> SINGLE_LINERS = new HashSet<Short>();
 64	static
 65	{
 66		SINGLE_LINERS.add(Terminals.ELSE);
 67		SINGLE_LINERS.add(Terminals.FUNC_ARROW);
 68		SINGLE_LINERS.add(Terminals.BOUND_FUNC_ARROW);
 69		SINGLE_LINERS.add(Terminals.TRY);
 70		SINGLE_LINERS.add(Terminals.FINALLY);
 71		SINGLE_LINERS.add(Terminals.THEN);
 72	}
 73
 74	private static Set<Short> SINGLE_CLOSERS = new HashSet<Short>();
 75	static
 76	{
 77		SINGLE_CLOSERS.add(Terminals.TERMINATOR);
 78		SINGLE_CLOSERS.add(Terminals.CATCH);
 79		SINGLE_CLOSERS.add(Terminals.FINALLY);
 80		SINGLE_CLOSERS.add(Terminals.ELSE);
 81		SINGLE_CLOSERS.add(Terminals.OUTDENT);
 82		SINGLE_CLOSERS.add(Terminals.LEADING_WHEN);
 83	}
 84
 85	private static Set<Short> IMPLICIT_FUNC = new HashSet<Short>();
 86	static
 87	{
 88		IMPLICIT_FUNC.add(Terminals.IDENTIFIER);
 89		IMPLICIT_FUNC.add(Terminals.SUPER);
 90		IMPLICIT_FUNC.add(Terminals.RPAREN);
 91		IMPLICIT_FUNC.add(Terminals.CALL_END);
 92		IMPLICIT_FUNC.add(Terminals.RBRACKET);
 93		IMPLICIT_FUNC.add(Terminals.INDEX_END);
 94		IMPLICIT_FUNC.add(Terminals.AT_SIGIL);
 95		IMPLICIT_FUNC.add(Terminals.THIS);
 96	}
 97
 98	private static Set<Short> IMPLICIT_CALL = new HashSet<Short>();
 99	static
100	{
101		IMPLICIT_CALL.add(Terminals.IDENTIFIER);
102		IMPLICIT_CALL.add(Terminals.NUMBER);
103		IMPLICIT_CALL.add(Terminals.STRING);
104		IMPLICIT_CALL.add(Terminals.JS);
105		IMPLICIT_CALL.add(Terminals.REGEX);
106		IMPLICIT_CALL.add(Terminals.NEW);
107		IMPLICIT_CALL.add(Terminals.PARAM_START);
108		IMPLICIT_CALL.add(Terminals.CLASS);
109		IMPLICIT_CALL.add(Terminals.IF);
110		IMPLICIT_CALL.add(Terminals.TRY);
111		IMPLICIT_CALL.add(Terminals.SWITCH);
112		IMPLICIT_CALL.add(Terminals.THIS);
113		IMPLICIT_CALL.add(Terminals.BOOL);
114		IMPLICIT_CALL.add(Terminals.UNARY);
115		IMPLICIT_CALL.add(Terminals.SUPER);
116		IMPLICIT_CALL.add(Terminals.AT_SIGIL);
117		IMPLICIT_CALL.add(Terminals.FUNC_ARROW);
118		IMPLICIT_CALL.add(Terminals.BOUND_FUNC_ARROW);
119		IMPLICIT_CALL.add(Terminals.LBRACKET);
120		IMPLICIT_CALL.add(Terminals.LPAREN);
121		IMPLICIT_CALL.add(Terminals.LCURLY);
122		IMPLICIT_CALL.add(Terminals.MINUS_MINUS);
123		IMPLICIT_CALL.add(Terminals.PLUS_PLUS);
124	}
125
126	private static Set<Short> IMPLICIT_UNSPACED_CALL = new HashSet<Short>();
127	static
128	{
129		IMPLICIT_UNSPACED_CALL.add(Terminals.PLUS);
130		IMPLICIT_UNSPACED_CALL.add(Terminals.MINUS);
131	}
132
133	private static Set<Short> IMPLICIT_BLOCK = new HashSet<Short>();
134	static
135	{
136		IMPLICIT_BLOCK.add(Terminals.FUNC_ARROW);
137		IMPLICIT_BLOCK.add(Terminals.BOUND_FUNC_ARROW);
138		IMPLICIT_BLOCK.add(Terminals.LCURLY);
139		IMPLICIT_BLOCK.add(Terminals.LBRACKET);
140		IMPLICIT_BLOCK.add(Terminals.COMMA);
141	}
142
143	private static Set<Short> IMPLICIT_END = new HashSet<Short>();
144	static
145	{
146		IMPLICIT_END.add(Terminals.POST_IF);
147		IMPLICIT_END.add(Terminals.FOR);
148		IMPLICIT_END.add(Terminals.WHILE);
149		IMPLICIT_END.add(Terminals.UNTIL);
150		IMPLICIT_END.add(Terminals.WHEN);
151		IMPLICIT_END.add(Terminals.BY);
152		IMPLICIT_END.add(Terminals.LOOP);
153		IMPLICIT_END.add(Terminals.TERMINATOR);
154		IMPLICIT_END.add(Terminals.INDENT);
155	}
156
157	private static Set<Short> LINEBREAKS = new HashSet<Short>();
158	static
159	{
160		LINEBREAKS.add(Terminals.TERMINATOR);
161		LINEBREAKS.add(Terminals.INDENT);
162		LINEBREAKS.add(Terminals.OUTDENT);
163	}
164
165	private static Set<Short> CHECK_FOR_IMPLICIT_INDENTATION = new HashSet<Short>();
166	static
167	{
168		CHECK_FOR_IMPLICIT_INDENTATION.add(Terminals.OUTDENT);
169		CHECK_FOR_IMPLICIT_INDENTATION.add(Terminals.TERMINATOR);
170		CHECK_FOR_IMPLICIT_INDENTATION.add(Terminals.FINALLY);
171	}
172
173	private static Set<Short> IMPLICIT_PARENS_CHECK_1 = new HashSet<Short>();
174	static
175	{
176		IMPLICIT_PARENS_CHECK_1.add(Terminals.IF);
177		IMPLICIT_PARENS_CHECK_1.add(Terminals.ELSE);
178		IMPLICIT_PARENS_CHECK_1.add(Terminals.FUNC_ARROW);
179		IMPLICIT_PARENS_CHECK_1.add(Terminals.BOUND_FUNC_ARROW);
180	}
181
182	private static Set<Short> IMPLICIT_PARENS_CHECK_2 = new HashSet<Short>();
183	static
184	{
185		IMPLICIT_PARENS_CHECK_2.add(Terminals.DOT);
186		IMPLICIT_PARENS_CHECK_2.add(Terminals.QUESTION_DOT);
187		IMPLICIT_PARENS_CHECK_2.add(Terminals.DOUBLE_COLON);
188	}
189
190	private static Set<Short> IMPLICIT_BRACES = new HashSet<Short>();
191	static
192	{
193		IMPLICIT_BRACES.add(Terminals.IDENTIFIER);
194		IMPLICIT_BRACES.add(Terminals.NUMBER);
195		IMPLICIT_BRACES.add(Terminals.STRING);
196		IMPLICIT_BRACES.add(Terminals.AT_SIGIL);
197		IMPLICIT_BRACES.add(Terminals.TERMINATOR);
198		IMPLICIT_BRACES.add(Terminals.OUTDENT);
199	}
200
201	private List<CoffeeSymbol> fTokens;
202	private boolean seenSingle;
203
204	List<CoffeeSymbol> rewrite(List<CoffeeSymbol> tokens) throws SyntaxError
205	{
206		this.fTokens = tokens;
207		removeLeadingNewlines();
208		removeMidExpressionNewlines();
209		closeOpenCalls();
210		closeOpenIndexes();
211		addImplicitIndentation();
212		tagPostfixConditionals();
213		addImplicitBraces();
214		addImplicitParentheses();
215		ensureBalance(BALANCED_PAIRS);
216		rewriteClosingParens();
217
218		return this.fTokens;
219	}
220
221	private void removeLeadingNewlines()
222	{
223		// Remove leading TERMINATORs
224		while (!fTokens.isEmpty())
225		{
226			CoffeeSymbol sym = fTokens.get(0);
227			if (Terminals.TERMINATOR != sym.getId())
228			{
229				break;
230			}
231			fTokens.remove(0);
232		}
233	}
234
235	private void removeMidExpressionNewlines()
236	{
237		for (int i = 0; i < this.fTokens.size();)
238		{
239			CoffeeSymbol sym = this.fTokens.get(i);
240			CoffeeSymbol next = null;
241			if (i + 1 < this.fTokens.size())
242			{
243				next = this.fTokens.get(i + 1);
244			}
245			if (sym.getId() == Terminals.TERMINATOR && next != null && EXPRESSION_CLOSE.contains(next.getId()))
246			{
247				this.fTokens.remove(i);
248				continue;
249			}
250			i++;
251		}
252	}
253
254	private void closeOpenCalls()
255	{
256		// scanTokens
257		for (int i = 0; i < this.fTokens.size(); i++)
258		{
259			CoffeeSymbol sym = this.fTokens.get(i);
260			if (Terminals.CALL_START == sym.getId())
261			{
262				// detectEnd
263				int levels = 0;
264				for (int j = i + 1; j < this.fTokens.size(); j++)
265				{
266					CoffeeSymbol token = this.fTokens.get(j);
267					if (levels == 0
268							&& ((token.getId() == Terminals.RPAREN || token.getId() == Terminals.CALL_END) || (token
269									.getId() == Terminals.OUTDENT && this.fTokens.get(j - 1).getId() == Terminals.RPAREN)))
270					{
271						int index = token.getId() == Terminals.OUTDENT ? j - 1 : j;
272						this.fTokens.get(index).setId(Terminals.CALL_END);
273						break;
274					}
275					if (token == null || levels < 0)
276					{
277						int index = token.getId() == Terminals.OUTDENT ? j - 2 : j - 1;
278						this.fTokens.get(index).setId(Terminals.CALL_END);
279						break;
280					}
281					if (EXPRESSION_START.contains(token.getId()))
282					{
283						levels++;
284					}
285					else if (EXPRESSION_END.contains(token.getId()))
286					{
287						levels--;
288					}
289				}
290			}
291		}
292	}
293
294	private void closeOpenIndexes()
295	{
296		// scanTokens
297		for (int i = 0; i < this.fTokens.size(); i++)
298		{
299			CoffeeSymbol sym = this.fTokens.get(i);
300			if (Terminals.INDEX_START == sym.getId())
301			{
302				// detectEnd
303				int levels = 0;
304				for (int j = i + 1; j < this.fTokens.size(); j++)
305				{
306					CoffeeSymbol token = this.fTokens.get(j);
307					if (levels == 0 && (token.getId() == Terminals.RBRACKET || token.getId() == Terminals.INDEX_END))
308					{
309						token.setId(Terminals.INDEX_END);
310						break;
311					}
312					if (token == null || levels < 0)
313					{
314						token.setId(Terminals.INDEX_END);
315						break;
316					}
317					if (EXPRESSION_START.contains(token.getId()))
318					{
319						levels++;
320					}
321					else if (EXPRESSION_END.contains(token.getId()))
322					{
323						levels--;
324					}
325				}
326			}
327		}
328	}
329
330	private void addImplicitBraces() throws SyntaxError
331	{
332		Stack<CoffeeSymbol> stack = new Stack<CoffeeSymbol>();
333		// scanTokens
334		for (int i = 0; i < this.fTokens.size();)
335		{
336			CoffeeSymbol token = this.fTokens.get(i);
337			if (EXPRESSION_START.contains(token.getId()))
338			{
339				short id;
340				if (Terminals.INDENT == token.getId() && i > 0 && Terminals.LCURLY == this.fTokens.get(i - 1).getId())
341				{
342					id = Terminals.LCURLY;
343				}
344				else
345				{
346					id = token.getId();
347				}
348
349				stack.add(new CoffeeSymbol(id, i));
350				i++;
351				continue;
352			}
353			if (EXPRESSION_END.contains(token.getId()))
354			{
355				if (stack.isEmpty())
356				{
357					errorAtToken(MessageFormat.format("Closing {0} without opening pair",
358							getTerminalNameForShort(token.getId())), i);
359				}
360				else
361				{
362					stack.pop();
363					i++;
364					continue;
365				}
366			}
367
368			short endOfStack = -1;
369			if (!stack.isEmpty())
370			{
371				endOfStack = stack.get(stack.size() - 1).getId();
372			}
373			if (!(Terminals.COLON == token.getId() && ((i >= 2 && Terminals.COLON == this.fTokens.get(i - 2).getId()) || (Terminals.LCURLY != endOfStack))))
374			{
375				i++;
376				continue;
377			}
378			stack.push(new CoffeeSymbol(Terminals.LCURLY, "{"));
379
380			int idx = ((i >= 2 && Terminals.AT_SIGIL == this.fTokens.get(i - 2).getId()) ? i - 2 : i - 1);
381			while (idx >= 2 && this.fTokens.get(idx - 2).getId() == Terminals.HERECOMMENT)
382			{
383				idx -= 2;
384			}
385			// Grab the end of the token before this implicit curly, and use that as our offset
386			int offsetToUse = 0;
387			if (idx >= 1)
388			{
389				CoffeeSymbol replacing = this.fTokens.get(idx - 1);
390				offsetToUse = replacing.getEnd();
391			}
392			CoffeeSymbol tok = new CoffeeSymbol(Terminals.LCURLY, offsetToUse, offsetToUse, "{");
393			tok.generated = true;
394			this.fTokens.add(idx, tok);
395
396			// detectEnd
397			int levels = 0;
398			for (int j = i + 2; j < this.fTokens.size(); j++)
399			{
400				CoffeeSymbol innerToken = this.fTokens.get(j);
401				if (levels == 0 && addImplicitBrace(innerToken, j))
402				{
403					CoffeeSymbol toAdd = new CoffeeSymbol(Terminals.RCURLY, innerToken.getStart(),
404							innerToken.getStart(), "}");
405					toAdd.generated = true;
406					this.fTokens.add(j, toAdd);
407					break;
408				}
409				if (innerToken == null || levels < 0)
410				{
411					CoffeeSymbol toAdd = new CoffeeSymbol(Terminals.RCURLY, innerToken.getStart(),
412							innerToken.getStart(), "}");
413					toAdd.generated = true;
414					this.fTokens.add(j, toAdd);
415					break;
416				}
417				if (EXPRESSION_START.contains(innerToken.getId()))
418				{
419					levels++;
420				}
421				else if (EXPRESSION_END.contains(innerToken.getId()))
422				{
423					levels--;
424				}
425			}
426			i += 2;
427		}
428	}
429
430	private boolean addImplicitBrace(CoffeeSymbol token, int i)
431	{
432		Short oneId = -1;
433		if (i + 1 < this.fTokens.size())
434		{
435			oneId = this.fTokens.get(i + 1).getId();
436		}
437		if (Terminals.HERECOMMENT == oneId)
438		{
439			return false;
440		}
441		short tag = token.getId();
442		Short twoId = -2;
443		if (i + 2 < this.fTokens.size())
444		{
445			twoId = this.fTokens.get(i + 2).getId();
446		}
447		Short threeId = -3;
448		if (i + 3 < this.fTokens.size())
449		{
450			threeId = this.fTokens.get(i + 3).getId();
451		}
452		// @formatter:off
453		return (
454				(Terminals.TERMINATOR == tag || Terminals.OUTDENT == tag) &&
455				!(Terminals.COLON == twoId || Terminals.AT_SIGIL == oneId && Terminals.COLON == threeId)
456				)
457				||
458				(Terminals.COMMA == tag && !IMPLICIT_BRACES.contains(oneId));
459		// @formatter:on
460	}
461
462	private List<CoffeeSymbol> indentation(CoffeeSymbol token)
463	{
464		List<CoffeeSymbol> symbols = new ArrayList<CoffeeSymbol>();
465		symbols.add(new CoffeeSymbol(Terminals.INDENT, token.getStart(), token.getStart(), 2));
466		symbols.add(new CoffeeSymbol(Terminals.OUTDENT, token.getStart(), token.getStart(), 2));
467		return symbols;
468	}
469
470	private void addImplicitIndentation()
471	{
472		// scanTokens
473		for (int i = 0; i < this.fTokens.size();)
474		{
475			CoffeeSymbol token = this.fTokens.get(i);
476			short tag = token.getId();
477			if (Terminals.TERMINATOR == tag && i + 1 < this.fTokens.size()
478					&& Terminals.THEN == this.fTokens.get(i + 1).getId())
479			{
480				this.fTokens.remove(i);
481				continue;
482			}
483			if (Terminals.ELSE == tag && Terminals.OUTDENT != this.fTokens.get(i - 1).getId())
484			{
485				this.fTokens.addAll(i, indentation(token));
486				i += 2;
487				continue;
488			}
489
490			if (Terminals.CATCH == tag && CHECK_FOR_IMPLICIT_INDENTATION.contains(this.fTokens.get(i + 2).getId()))
491			{
492				this.fTokens.addAll(i + 2, indentation(token));
493				i += 4;
494				continue;
495			}
496
497			if (SINGLE_LINERS.contains(tag) && Terminals.INDENT != this.fTokens.get(i + 1).getId()
498					&& (!(Terminals.ELSE == tag && Terminals.IF == this.fTokens.get(i + 1).getId())))
499			{
500				short starter = tag;
501				List<CoffeeSymbol> indents = indentation(token);
502				CoffeeSymbol indent = indents.get(0);
503				CoffeeSymbol outdent = indents.get(1);
504				indent.fromThen = (Terminals.THEN == starter);
505				indent.generated = true;
506				outdent.generated = true;
507				this.fTokens.add(i + 1, indent);
508				// detectEnd
509				int levels = 0;
510				for (int j = i + 2; j < this.fTokens.size(); j++)
511				{
512					CoffeeSymbol innerToken = this.fTokens.get(j);
513					if (levels == 0 && addImplicitIndent(innerToken, starter, j))
514					{
515						int index = j;
516						if (Terminals.COMMA == this.fTokens.get(j - 1).getId())
517						{
518							index = j - 1;
519						}
520						// Fix the outdent offsets!
521						outdent.setLocation(innerToken.getStart(), innerToken.getStart());
522						this.fTokens.add(index, outdent);
523						break;
524					}
525					if (innerToken == null || levels < 0)
526					{
527						int index = j;
528						if (Terminals.COMMA == this.fTokens.get(j - 1).getId())
529						{
530							index = j - 1;
531						}
532						// Fix the outdent offsets!
533						outdent.setLocation(innerToken.getEnd(), innerToken.getEnd());
534						this.fTokens.add(index, outdent);
535						break;
536					}
537					if (EXPRESSION_START.contains(innerToken.getId()))
538					{
539						levels++;
540					}
541					else if (EXPRESSION_END.contains(innerToken.getId()))
542					{
543						levels--;
544					}
545				}
546
547				if (Terminals.THEN == tag)
548				{
549					this.fTokens.remove(i);
550				}
551				i++;
552				continue;
553			}
554			i++;
555		}
556	}
557
558	private boolean addImplicitIndent(CoffeeSymbol token, short starter, int i)
559	{
560		Set<Short> toCheck = new HashSet<Short>();
561		toCheck.add(Terminals.IF);
562		toCheck.add(Terminals.THEN);
563		return (!";".equals(token.getValue()) && SINGLE_CLOSERS.contains(token.getId()) && !(Terminals.ELSE == token
564				.getId() && !toCheck.contains(starter)));
565	}
566
567	private void addImplicitParentheses()
568	{
569		boolean noCall = false;
570		// scanTokens
571		for (int i = 0; i < this.fTokens.size();)
572		{
573			CoffeeSymbol token = this.fTokens.get(i);
574			short tag = token.getId();
575			if (Terminals.CLASS == tag || Terminals.IF == tag)
576			{
577				noCall = true;
578			}
579			CoffeeSymbol prev = null;
580			if (i - 1 >= 0)
581			{
582				prev = this.fTokens.get(i - 1);
583			}
584			CoffeeSymbol next = null;
585			if (i + 1 < this.fTokens.size())
586			{
587				next = this.fTokens.get(i + 1);
588			}
589			boolean callObject = (!noCall && Terminals.INDENT == tag && next != null && next.generated
590					&& Terminals.LCURLY == next.getId() && prev != null && IMPLICIT_FUNC.contains(prev.getId()));
591			this.seenSingle = false;
592			if (LINEBREAKS.contains(tag))
593			{
594				noCall = false;
595			}
596			if (prev != null && !prev.spaced && Terminals.QUESTION == tag)
597			{
598				token.call = true;
599			}
600			if (token.fromThen)
601			{
602				i++;
603				continue;
604			}
605
606			if (!(callObject || (prev != null ? prev.spaced : false)
607					&& (prev.call || IMPLICIT_FUNC.contains(prev.getId()))
608					&& (IMPLICIT_CALL.contains(tag) || !(token.spaced || token.newLine)
609							&& IMPLICIT_UNSPACED_CALL.contains(tag))))
610			{
611
612				i++;
613				continue;
614			}
615
616			this.fTokens.add(i, new CoffeeSymbol(Terminals.CALL_START, token.getStart(), token.getStart(), "("));
617
618			// detectEnd
619			int levels = 0;
620			for (int j = i + 1; j < this.fTokens.size(); j++)
621			{
622				CoffeeSymbol innerToken = this.fTokens.get(j);
623
624				if (levels == 0 && addImplicitParens(innerToken, j))
625				{
626					int idx = (Terminals.OUTDENT == innerToken.getId() ? j + 1 : j);
627					this.fTokens.add(idx, new CoffeeSymbol(Terminals.CALL_END, innerToken.getStart(),
628							innerToken.getStart(), ")"));
629					break;
630				}
631				if (innerToken == null || levels < 0)
632				{
633					int idx = (Terminals.OUTDENT == innerToken.getId() ? j + 1 : j);
634					this.fTokens.add(idx, new CoffeeSymbol(Terminals.CALL_END, innerToken.getStart(),
635							innerToken.getStart(), ")"));
636					break;
637				}
638				if (EXPRESSION_START.contains(innerToken.getId()))
639				{
640					levels++;
641				}
642				else if (EXPRESSION_END.contains(innerToken.getId()))
643				{
644					levels--;
645				}
646			}
647
648			if (Terminals.QUESTION == prev.getId())
649			{
650				prev.setId(Terminals.FUNC_EXIST);
651			}
652			i += 2;
653		}
654	}
655
656	private boolean addImplicitParens(CoffeeSymbol token, int i)
657	{
658		if (!seenSingle && token.fromThen)
659		{
660			return true;
661		}
662		short tag = token.getId();
663		if (IMPLICIT_PARENS_CHECK_1.contains(tag))
664		{
665			seenSingle = true;
666		}
667
668		CoffeeSymbol prev = null;
669		if (i - 1 >= 0)
670		{
671			prev = this.fTokens.get(i - 1);
672		}
673
674		if (IMPLICIT_PARENS_CHECK_2.contains(tag) && prev != null && Terminals.OUTDENT == prev.getId())
675		{
676			return true;
677		}
678		CoffeeSymbol post = null;
679		if (i + 1 < this.fTokens.size())
680		{
681			post = this.fTokens.get(i + 1);
682		}
683
684		return (!token.generated && Terminals.COMMA != prev.getId() && IMPLICIT_END.contains(tag) && (Terminals.INDENT != tag || (Terminals.CLASS != this.fTokens
685				.get(i - 2).getId() && !IMPLICIT_BLOCK.contains(prev.getId()) && !(post != null && post.generated && Terminals.LCURLY == post
686				.getId()))));
687	}
688
689	private void tagPostfixConditionals()
690	{
691		// scanTokens
692		for (int i = 0; i < this.fTokens.size();)
693		{
694			CoffeeSymbol token = this.fTokens.get(i);
695			if (Terminals.IF != token.getId())
696			{
697				i++;
698				continue;
699			}
700
701			// detectEnd
702			int levels = 0;
703			for (int j = i + 1; j < this.fTokens.size(); j++)
704			{
705				CoffeeSymbol innerToken = this.fTokens.get(j);
706
707				if (levels == 0
708						&& (Terminals.TERMINATOR == innerToken.getId() || Terminals.INDENT == innerToken.getId()))
709				{
710					if (Terminals.INDENT != innerToken.getId())
711					{
712						token.setId(Terminals.POST_IF);
713					}
714					break;
715				}
716				if (innerToken == null || levels < 0)
717				{
718					if (Terminals.INDENT != innerToken.getId())
719					{
720						token.setId(Terminals.POST_IF);
721					}
722					break;
723				}
724				if (EXPRESSION_START.contains(innerToken.getId()))
725				{
726					levels++;
727				}
728				else if (EXPRESSION_END.contains(innerToken.getId()))
729				{
730					levels--;
731				}
732			}
733			i++;
734		}
735	}
736
737	private void ensureBalance(Map<Short, Short> pairs) throws SyntaxError
738	{
739		Map<Short, Integer> levels = new HashMap<Short, Integer>();
740		Map<Short, Integer> openLine = new HashMap<Short, Integer>();
741		Map<Integer, Integer> offsetToTokenIndex = new HashMap<Integer, Integer>();
742		for (int i = 0; i < fTokens.size(); i++)
743		{
744			CoffeeSymbol token = fTokens.get(i);
745			for (Map.Entry<Short, Short> pair : pairs.entrySet())
746			{
747				short open = pair.getKey();
748				short close = pair.getValue();
749				if (!levels.containsKey(open))
750				{
751					levels.put(open, 0);
752				}
753				if (open == token.getId())
754				{
755					int level = levels.get(open);
756					if (level == 0)
757					{
758						// FIXME Put line as value, not offset!
759						openLine.put(open, token.getStart());
760						offsetToTokenIndex.put(token.getStart(), i);
761					}
762					level++;
763					levels.put(open, level);
764				}
765				else if (close == token.getId())
766				{
767					int level = levels.get(open);
768					level--;
769					levels.put(open, level);
770					if (level < 0)
771					{
772						// FIXME When we store lines, spit out line number here, not offset
773						errorAtToken(MessageFormat.format("too many {0} at offset {1}",
774								getTerminalNameForShort(close), token.getStart()), i);
775					}
776				}
777			}
778		}
779
780		for (Map.Entry<Short, Integer> entry : levels.entrySet())
781		{
782			Integer level = entry.getValue();
783			if (level > 0)
784			{
785				Short open = entry.getKey();
786				int offset = openLine.get(open);
787				int tokenIndex = offsetToTokenIndex.get(offset);
788				errorAtToken(MessageFormat.format("unclosed {0} at offset {1}",
789						getTerminalNameForShort(open), offset), tokenIndex);
790			}
791		}
792	}
793
794	private String getTerminalNameForShort(Short id)
795	{
796		return Terminals.getNameForValue(id);
797	}
798
799	private void rewriteClosingParens()
800	{
801		Stack<CoffeeSymbol> stack = new Stack<CoffeeSymbol>();
802		Map<Short, Integer> debt = new HashMap<Short, Integer>();
803		for (Short key : INVERSES.keySet())
804		{
805			debt.put(key, 0);
806		}
807		// scanTokens
808		for (int i = 0; i < this.fTokens.size();)
809		{
810			CoffeeSymbol token = this.fTokens.get(i);
811			short tag = token.getId();
812			if (EXPRESSION_START.contains(tag))
813			{
814				stack.push(token);
815				i++;
816				continue;
817			}
818			if (!EXPRESSION_END.contains(tag))
819			{
820				i++;
821				continue;
822			}
823			Short inv = INVERSES.get(tag);
824			int invValue = debt.get(inv);
825			if (invValue > 0)
826			{
827				invValue--;
828				debt.put(inv, invValue);
829				this.fTokens.remove(i);
830				continue;
831			}
832			CoffeeSymbol match = stack.pop();
833			short mtag = match.getId();
834			short oppos = INVERSES.get(mtag);
835			if (tag == oppos)
836			{
837				i++;
838				continue;
839			}
840
841			int mtagValue = debt.get(mtag);
842			mtagValue++;
843			debt.put(mtag, mtagValue);
844
845			// Use start offset of next token as our start and end offset
846			CoffeeSymbol val = new CoffeeSymbol(oppos, token.getStart(), token.getStart(),
847					Terminals.INDENT == mtag ? match.getValue() : Terminals.getValue(oppos));
848
849			if (mtag == this.fTokens.get(i + 2).getId())
850			{
851				this.fTokens.add(i + 3, val);
852				stack.push(match);
853			}
854			else
855			{
856				this.fTokens.add(i, val);
857			}
858			i++;
859		}
860	}
861	
862	/**
863	 * Remove all tokens starting at <var>tokenIndex</var>, then throw an exception with <var>errorMessage</var>
864	 * @throws SyntaxError 
865	 */
866	private void errorAtToken(String errorMessage, int tokenIndex) throws SyntaxError
867	{
868		fTokens.subList(tokenIndex, fTokens.size()).clear();
869		throw new SyntaxError(errorMessage);
870	}
871}