/lib/antlr-2.7.5/antlr/CppCodeGenerator.java

Large files files are truncated, but you can click here to view the full file

package antlr;

/* ANTLR Translator Generator
 * Project led by Terence Parr at http://www.jGuru.com
 * Software rights: http://www.antlr.org/license.html
 *
 * $Id: //depot/code/org.antlr/release/antlr-2.7.5/antlr/CppCodeGenerator.java#1 $
 */

// C++ code generator by Pete Wells: pete@yamuna.demon.co.uk
// #line generation contributed by: Ric Klaren <klaren@cs.utwente.nl>

import java.util.Enumeration;
import java.util.Hashtable;
import antlr.collections.impl.BitSet;
import antlr.collections.impl.Vector;
import java.io.PrintWriter; //SAS: changed for proper text file io
import java.io.IOException;
import java.io.FileWriter;

/** Generate MyParser.cpp, MyParser.hpp, MyLexer.cpp, MyLexer.hpp
 * and MyParserTokenTypes.hpp
 */
public class CppCodeGenerator extends CodeGenerator {
	boolean DEBUG_CPP_CODE_GENERATOR = false;
	// non-zero if inside syntactic predicate generation
	protected int syntacticPredLevel = 0;

	// Are we generating ASTs (for parsers and tree parsers) right now?
	protected boolean genAST = false;

	// Are we saving the text consumed (for lexers) right now?
	protected boolean saveText = false;

	// Generate #line's
	protected boolean genHashLines = true;
	// Generate constructors or not
	protected boolean noConstructors = false;

	// Used to keep track of lineno in output
	protected int outputLine;
	protected String outputFile;

	// Grammar parameters set up to handle different grammar classes.
	// These are used to get instanceof tests out of code generation
	boolean usingCustomAST = false;
	String labeledElementType;
	String labeledElementASTType; // mostly the same as labeledElementType except in parsers
	String labeledElementASTInit;
	String labeledElementInit;
	String commonExtraArgs;
	String commonExtraParams;
	String commonLocalVars;
	String lt1Value;
	String exceptionThrown;
	String throwNoViable;

	// Tracks the rule being generated.  Used for mapTreeId
	RuleBlock currentRule;
	// Tracks the rule or labeled subrule being generated.  Used for AST generation.
	String currentASTResult;
	// Mapping between the ids used in the current alt, and the
	// names of variables used to represent their AST values.
	Hashtable treeVariableMap = new Hashtable();

	/** Used to keep track of which AST variables have been defined in a rule
	 * (except for the #rule_name and #rule_name_in var's
	 */
	Hashtable declaredASTVariables = new Hashtable();

	// Count of unnamed generated variables
	int astVarNumber = 1;
	// Special value used to mark duplicate in treeVariableMap
	protected static final String NONUNIQUE = new String();

	public static final int caseSizeThreshold = 127; // ascii is max

	private Vector semPreds;

	// Used to keep track of which (heterogeneous AST types are used)
	// which need to be set in the ASTFactory of the generated parser
	private Vector astTypes;

	private static String namespaceStd   = "ANTLR_USE_NAMESPACE(std)";
	private static String namespaceAntlr = "ANTLR_USE_NAMESPACE(antlr)";
	private static NameSpace nameSpace = null;

	private static final String preIncludeCpp  = "pre_include_cpp";
	private static final String preIncludeHpp  = "pre_include_hpp";
	private static final String postIncludeCpp = "post_include_cpp";
	private static final String postIncludeHpp = "post_include_hpp";

	/** Create a C++ code-generator using the given Grammar.
	 * The caller must still call setTool, setBehavior, and setAnalyzer
	 * before generating code.
	 */
	public CppCodeGenerator() {
		super();
		charFormatter = new CppCharFormatter();
	}
	/** Adds a semantic predicate string to the sem pred vector
	    These strings will be used to build an array of sem pred names
	    when building a debugging parser.  This method should only be
	    called when the debug option is specified
	 */
	protected int addSemPred(String predicate) {
		semPreds.appendElement(predicate);
		return semPreds.size()-1;
	}
	public void exitIfError()
	{
		if (antlrTool.hasError())
		{
			antlrTool.fatalError("Exiting due to errors.");
		}
	}
	protected int countLines( String s )
	{
		int lines = 0;
		for( int i = 0; i < s.length(); i++ )
		{
			if( s.charAt(i) == '\n' )
				lines++;
		}
		return lines;
	}
	/** Output a String to the currentOutput stream.
	 * Ignored if string is null.
	 * @param s The string to output
	 */
	protected void _print(String s)
	{
		if (s != null)
		{
			outputLine += countLines(s);
			currentOutput.print(s);
		}
	}
	/** Print an action without leading tabs, attempting to
	 * preserve the current indentation level for multi-line actions
	 * Ignored if string is null.
	 * @param s The action string to output
	 */
	protected void _printAction(String s)
	{
		if (s != null)
		{
			outputLine += countLines(s)+1;
			super._printAction(s);
		}
	}
	/** Print an action stored in a token surrounded by #line stuff */
	public void printAction(Token t)
	{
		if (t != null)
		{
			genLineNo(t.getLine());
			printTabs();
			_printAction(processActionForSpecialSymbols(t.getText(), t.getLine(),
																null, null) );
			genLineNo2();
		}
	}
	/** Print a header action by #line stuff also process any tree construction
	 * @param name The name of the header part
	 */
	public void printHeaderAction(String name)
	{
		Token a = (antlr.Token)behavior.headerActions.get(name);
		if (a != null)
		{
			genLineNo(a.getLine());
			println(processActionForSpecialSymbols(a.getText(), a.getLine(),
																null, null) );
			genLineNo2();
		}
	}
	/** Output a String followed by newline, to the currentOutput stream.
	 * Ignored if string is null.
	 * @param s The string to output
	 */
	protected void _println(String s) {
		if (s != null) {
			outputLine += countLines(s)+1;
			currentOutput.println(s);
		}
	}
	/** Output tab indent followed by a String followed by newline,
	 * to the currentOutput stream.  Ignored if string is null.
	 * @param s The string to output
	 */
	protected void println(String s) {
		if (s != null) {
			printTabs();
			outputLine += countLines(s)+1;
			currentOutput.println(s);
		}
	}

	/** Generate a #line or // line depending on options */
	public void genLineNo(int line) {
		if ( line == 0 ) {
			line++;
		}
		if( genHashLines )
			_println("#line "+line+" \""+antlrTool.fileMinusPath(antlrTool.grammarFile)+"\"");
	}

	/** Generate a #line or // line depending on options */
	public void genLineNo(GrammarElement el)
	{
		if( el != null )
			genLineNo(el.getLine());
	}
	/** Generate a #line or // line depending on options */
	public void genLineNo(Token t)
	{
		if (t != null)
			genLineNo(t.getLine());
	}
	/** Generate a #line or // line depending on options */
	public void genLineNo2()
	{
		if( genHashLines )
		{
			_println("#line "+(outputLine+1)+" \""+outputFile+"\"");
		}
	}
	/// Bound safe isDigit
	private boolean charIsDigit( String s, int i )
	{
		return (i < s.length()) && Character.isDigit(s.charAt(i));
	}
	/** Normalize a string coming from antlr's lexer. E.g. translate java
	 * escapes to values. Check their size (multibyte) bomb out if they are
	 * multibyte (bit crude). Then reescape to C++ style things.
	 * Used to generate strings for match() and matchRange()
	 * @param lit the literal string
	 * @param isCharLiteral if it's for a character literal
	 *                       (enforced to be one length) and enclosed in '
	 * FIXME: bombing out on mb chars. Should be done in Lexer.
	 * FIXME: this is another horrible hack.
	 * FIXME: life would be soooo much easier if the stuff from the lexer was
	 * normalized in some way.
	 */
	private String convertJavaToCppString( String lit, boolean isCharLiteral )
	{
		// System.out.println("convertJavaToCppLiteral: "+lit);
		String ret = new String();
		String s = lit;

		int i = 0;
		int val = 0;

		if( isCharLiteral )	// verify & strip off quotes
		{
			if( ! lit.startsWith("'") || ! lit.endsWith("'") )
				antlrTool.error("Invalid character literal: '"+lit+"'");
		}
		else
		{
			if( ! lit.startsWith("\"") || ! lit.endsWith("\"") )
				antlrTool.error("Invalid character string: '"+lit+"'");
		}
		s = lit.substring(1,lit.length()-1);

		String prefix="";
		int maxsize = 255;
		if( grammar instanceof LexerGrammar )
		{
			// vocab size seems to be 1 bigger than it actually is
			maxsize = ((LexerGrammar)grammar).charVocabulary.size() - 1;
			if( maxsize > 255 )
				prefix= "L";
		}

		// System.out.println("maxsize "+maxsize+" prefix "+prefix);

		while ( i < s.length() )
		{
			if( s.charAt(i) == '\\' )
			{
				if( s.length() == i+1 )
					antlrTool.error("Invalid escape in char literal: '"+lit+"' looking at '"+s.substring(i)+"'");

				// deal with escaped junk
				switch ( s.charAt(i+1) ) {
				case 'a' :
					val = 7;
					i += 2;
					break;
				case 'b' :
					val = 8;
					i += 2;
					break;
				case 't' :
					val = 9;
					i += 2;
					break;
				case 'n' :
					val = 10;
					i += 2;
					break;
				case 'f' :
					val = 12;
					i += 2;
					break;
				case 'r' :
					val = 13;
					i += 2;
					break;
				case '"' :
				case '\'' :
				case '\\' :
					val = s.charAt(i+1);
					i += 2;
					break;

				case 'u' :
					// Unicode char \u1234
					if( i+5 < s.length() )
					{
						val = Character.digit(s.charAt(i+2), 16) * 16 * 16 * 16 +
							Character.digit(s.charAt(i+3), 16) * 16 * 16 +
							Character.digit(s.charAt(i+4), 16) * 16 +
							Character.digit(s.charAt(i+5), 16);
						i += 6;
					}
					else
						antlrTool.error("Invalid escape in char literal: '"+lit+"' looking at '"+s.substring(i)+"'");
					break;

				case '0' :					// \123
				case '1' :
				case '2' :
				case '3' :
					if( charIsDigit(s, i+2) )
					{
						if( charIsDigit(s, i+3) )
						{
							val = (s.charAt(i+1)-'0')*8*8 + (s.charAt(i+2)-'0')*8 +
								(s.charAt(i+3)-'0');
							i += 4;
						}
						else
						{
							val = (s.charAt(i+1)-'0')*8 + (s.charAt(i+2)-'0');
							i += 3;
						}
					}
					else
					{
						val = s.charAt(i+1)-'0';
						i += 2;
					}
					break;

				case '4' :
				case '5' :
				case '6' :
				case '7' :
					if ( charIsDigit(s, i+2) )
					{
						val = (s.charAt(i+1)-'0')*8 + (s.charAt(i+2)-'0');
						i += 3;
					}
					else
					{
						val = s.charAt(i+1)-'0';
						i += 2;
					}
				default:
					antlrTool.error("Unhandled escape in char literal: '"+lit+"' looking at '"+s.substring(i)+"'");
					val = 0;
				}
			}
			else
				val = s.charAt(i++);

			if( grammar instanceof LexerGrammar )
			{
				if( val > maxsize )			// abort if too big
				{
					String offender;
					if( ( 0x20 <= val ) && ( val < 0x7F ) )
						offender = charFormatter.escapeChar(val,true);
					else
						offender = "0x"+Integer.toString(val,16);

					antlrTool.error("Character out of range in "+(isCharLiteral?"char literal":"string constant")+": '"+s+"'");
					antlrTool.error("Vocabulary size: "+maxsize+" Character "+offender);
				}
			}

			if( isCharLiteral )
			{
				// we should be at end of char literal here..
				if( i != s.length() )
					antlrTool.error("Invalid char literal: '"+lit+"'");

				if( maxsize <= 255 )
				{
					if ( (val <= 255) && (val & 0x80) != 0 )
						// the joys of sign extension in the support lib *cough*
						// actually the support lib needs to be fixed but that's a bit
						// hairy too.
						ret = "static_cast<unsigned char>('"+charFormatter.escapeChar(val,true)+"')";
					else
						ret = "'"+charFormatter.escapeChar(val,true)+"'";
				}
				else
				{
					// so wchar_t is some implementation defined int like thing
					// so this may even lead to having 16 bit or 32 bit cases...
					// I smell some extra grammar options in the future :(
					ret = "L'"+charFormatter.escapeChar(val,true)+"'";
				}
			}
			else
				ret += charFormatter.escapeChar(val,true);
		}
		if( !isCharLiteral )
			ret = prefix+"\""+ret+"\"";
		return ret;
	}
	/** Generate the parser, lexer, treeparser, and token types in C++
	 */
	public void gen() {
		// Do the code generation
		try {
			// Loop over all grammars
			Enumeration grammarIter = behavior.grammars.elements();
			while (grammarIter.hasMoreElements()) {
				Grammar g = (Grammar)grammarIter.nextElement();
				if ( g.debuggingOutput ) {
					antlrTool.error(g.getFilename()+": C++ mode does not support -debug");
				}
				// Connect all the components to each other
				g.setGrammarAnalyzer(analyzer);
				g.setCodeGenerator(this);
				analyzer.setGrammar(g);
				// To get right overloading behavior across hetrogeneous grammars
				setupGrammarParameters(g);
				g.generate();
				exitIfError();
			}

			// Loop over all token managers (some of which are lexers)
			Enumeration tmIter = behavior.tokenManagers.elements();
			while (tmIter.hasMoreElements()) {
				TokenManager tm = (TokenManager)tmIter.nextElement();
				if (!tm.isReadOnly()) {
					// Write the token manager tokens as C++
					// this must appear before genTokenInterchange so that
					// labels are set on string literals
					genTokenTypes(tm);
					// Write the token manager tokens as plain text
					genTokenInterchange(tm);
				}
				exitIfError();
			}
		}
		catch (IOException e) {
			antlrTool.reportException(e, null);
		}
	}
	/** Generate code for the given grammar element.
	 * @param blk The {...} action to generate
	 */
	public void gen(ActionElement action) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("genAction("+action+")");
		if ( action.isSemPred ) {
			genSemPred(action.actionText, action.line);
		}
		else {
			if ( grammar.hasSyntacticPredicate ) {
				println("if ( inputState->guessing==0 ) {");
				tabs++;
			}

			ActionTransInfo tInfo = new ActionTransInfo();
			String actionStr = processActionForSpecialSymbols(action.actionText,
																			  action.getLine(),
																			  currentRule, tInfo);

			if ( tInfo.refRuleRoot!=null ) {
				// Somebody referenced "#rule", make sure translated var is valid
				// assignment to #rule is left as a ref also, meaning that assignments
				// with no other refs like "#rule = foo();" still forces this code to be
				// generated (unnecessarily).
				println(tInfo.refRuleRoot + " = "+labeledElementASTType+"(currentAST.root);");
			}

			// dump the translated action
			genLineNo(action);
			printAction(actionStr);
			genLineNo2();

			if ( tInfo.assignToRoot ) {
				// Somebody did a "#rule=", reset internal currentAST.root
				println("currentAST.root = "+tInfo.refRuleRoot+";");
				// reset the child pointer too to be last sibling in sibling list
				// now use if else in stead of x ? y : z to shut CC 4.2 up.
				println("if ( "+tInfo.refRuleRoot+"!="+labeledElementASTInit+" &&");
				tabs++;
				println(tInfo.refRuleRoot+"->getFirstChild() != "+labeledElementASTInit+" )");
				println("  currentAST.child = "+tInfo.refRuleRoot+"->getFirstChild();");
			  	tabs--;
				println("else");
				tabs++;
				println("currentAST.child = "+tInfo.refRuleRoot+";");
				tabs--;
				println("currentAST.advanceChildToEnd();");
			}

			if ( grammar.hasSyntacticPredicate ) {
				tabs--;
				println("}");
			}
		}
	}

	/** Generate code for the given grammar element.
	 * @param blk The "x|y|z|..." block to generate
	 */
	public void gen(AlternativeBlock blk) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("gen("+blk+")");
		println("{");
		genBlockPreamble(blk);
		genBlockInitAction(blk);

		// Tell AST generation to build subrule result
		String saveCurrentASTResult = currentASTResult;
		if (blk.getLabel() != null) {
			currentASTResult = blk.getLabel();
		}

		boolean ok = grammar.theLLkAnalyzer.deterministic(blk);

		CppBlockFinishingInfo howToFinish = genCommonBlock(blk, true);
		genBlockFinish(howToFinish, throwNoViable);

		println("}");

		// Restore previous AST generation
		currentASTResult = saveCurrentASTResult;
	}
	/** Generate code for the given grammar element.
	 * @param blk The block-end element to generate.  Block-end
	 * elements are synthesized by the grammar parser to represent
	 * the end of a block.
	 */
	public void gen(BlockEndElement end) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("genRuleEnd("+end+")");
	}
	/** Generate code for the given grammar element.
	 * Only called from lexer grammars.
	 * @param blk The character literal reference to generate
	 */
	public void gen(CharLiteralElement atom) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR )
			System.out.println("genChar("+atom+")");

		if ( ! (grammar instanceof LexerGrammar) )
			antlrTool.error("cannot ref character literals in grammar: "+atom);

		if ( atom.getLabel() != null ) {
			println(atom.getLabel() + " = " + lt1Value + ";");
		}

		boolean oldsaveText = saveText;
		saveText = saveText && atom.getAutoGenType() == GrammarElement.AUTO_GEN_NONE;

		// if in lexer and ! on element, save buffer index to kill later
		if ( !saveText||atom.getAutoGenType()==GrammarElement.AUTO_GEN_BANG )
			println("_saveIndex = text.length();");

		print(atom.not ? "matchNot(" : "match(");
		_print(convertJavaToCppString( atom.atomText, true ));
		_println(" /* charlit */ );");

		if ( !saveText || atom.getAutoGenType() == GrammarElement.AUTO_GEN_BANG )
			println("text.erase(_saveIndex);");      // kill text atom put in buffer

		saveText = oldsaveText;
	}
	/** Generate code for the given grammar element.
	 * Only called from lexer grammars.
	 * @param blk The character-range reference to generate
	 */
	public void gen(CharRangeElement r) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR )
			System.out.println("genCharRangeElement("+r.beginText+".."+r.endText+")");

		if ( ! (grammar instanceof LexerGrammar) )
			antlrTool.error("cannot ref character range in grammar: "+r);

		if ( r.getLabel() != null && syntacticPredLevel == 0) {
			println(r.getLabel() + " = " + lt1Value + ";");
		}
		// Correctly take care of saveIndex stuff...
		boolean save = ( grammar instanceof LexerGrammar &&
							  ( !saveText ||
								 r.getAutoGenType() == GrammarElement.AUTO_GEN_BANG )
						   );
		if (save)
         println("_saveIndex=text.length();");

		println("matchRange("+convertJavaToCppString(r.beginText,true)+
				  ","+convertJavaToCppString(r.endText,true)+");");

		if (save)
         println("text.erase(_saveIndex);");
	}
	/** Generate the lexer C++ files */
	public  void gen(LexerGrammar g) throws IOException {
		// If debugging, create a new sempred vector for this grammar
		if (g.debuggingOutput)
			semPreds = new Vector();

		if( g.charVocabulary.size() > 256 )
			antlrTool.warning(g.getFilename()+": Vocabularies of this size still experimental in C++ mode (vocabulary size now: "+g.charVocabulary.size()+")");

		setGrammar(g);
		if (!(grammar instanceof LexerGrammar)) {
			antlrTool.panic("Internal error generating lexer");
		}

		genBody(g);
		genInclude(g);
	}
	/** Generate code for the given grammar element.
	 * @param blk The (...)+ block to generate
	 */
	public void gen(OneOrMoreBlock blk) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("gen+("+blk+")");
		String label;
		String cnt;
		println("{ // ( ... )+");
		genBlockPreamble(blk);
		if ( blk.getLabel() != null ) {
			cnt = "_cnt_"+blk.getLabel();
		}
		else {
			cnt = "_cnt" + blk.ID;
		}
		println("int "+cnt+"=0;");
		if ( blk.getLabel() != null ) {
			label = blk.getLabel();
		}
		else {
			label = "_loop" + blk.ID;
		}

		println("for (;;) {");
		tabs++;
		// generate the init action for ()+ ()* inside the loop
		// this allows us to do usefull EOF checking...
		genBlockInitAction(blk);

		// Tell AST generation to build subrule result
		String saveCurrentASTResult = currentASTResult;
		if (blk.getLabel() != null) {
			currentASTResult = blk.getLabel();
		}

		boolean ok = grammar.theLLkAnalyzer.deterministic(blk);

		// generate exit test if greedy set to false
		// and an alt is ambiguous with exit branch
		// or when lookahead derived purely from end-of-file
		// Lookahead analysis stops when end-of-file is hit,
		// returning set {epsilon}.  Since {epsilon} is not
		// ambig with any real tokens, no error is reported
		// by deterministic() routines and we have to check
		// for the case where the lookahead depth didn't get
		// set to NONDETERMINISTIC (this only happens when the
		// FOLLOW contains real atoms + epsilon).
		boolean generateNonGreedyExitPath = false;
		int nonGreedyExitDepth = grammar.maxk;

		if ( !blk.greedy &&
			 blk.exitLookaheadDepth<=grammar.maxk &&
			 blk.exitCache[blk.exitLookaheadDepth].containsEpsilon() )
		{
			generateNonGreedyExitPath = true;
			nonGreedyExitDepth = blk.exitLookaheadDepth;
		}
		else if ( !blk.greedy &&
				  blk.exitLookaheadDepth==LLkGrammarAnalyzer.NONDETERMINISTIC )
		{
			generateNonGreedyExitPath = true;
		}

		// generate exit test if greedy set to false
		// and an alt is ambiguous with exit branch
		if ( generateNonGreedyExitPath ) {
			if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) {
				System.out.println("nongreedy (...)+ loop; exit depth is "+
								   blk.exitLookaheadDepth);
			}
			String predictExit =
				getLookaheadTestExpression(blk.exitCache,
										   nonGreedyExitDepth);
			println("// nongreedy exit test");
			println("if ( "+cnt+">=1 && "+predictExit+") goto "+label+";");
		}

		CppBlockFinishingInfo howToFinish = genCommonBlock(blk, false);
		genBlockFinish(
			howToFinish,
			"if ( "+cnt+">=1 ) { goto "+label+"; } else {" + throwNoViable + "}"
		);

		println(cnt+"++;");
		tabs--;
		println("}");
		println(label+":;");
		println("}  // ( ... )+");

		// Restore previous AST generation
		currentASTResult = saveCurrentASTResult;
	}
	/** Generate the parser C++ file */
	public void gen(ParserGrammar g) throws IOException {

		// if debugging, set up a new vector to keep track of sempred
		//   strings for this grammar
		if (g.debuggingOutput)
			semPreds = new Vector();

		setGrammar(g);
		if (!(grammar instanceof ParserGrammar)) {
			antlrTool.panic("Internal error generating parser");
		}

		genBody(g);
		genInclude(g);
	}
	/** Generate code for the given grammar element.
	 * @param blk The rule-reference to generate
	 */
	public void gen(RuleRefElement rr)
	{
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("genRR("+rr+")");
		RuleSymbol rs = (RuleSymbol)grammar.getSymbol(rr.targetRule);
		if (rs == null || !rs.isDefined())
		{
			// Is this redundant???
			antlrTool.error("Rule '" + rr.targetRule + "' is not defined", grammar.getFilename(), rr.getLine(), rr.getColumn());
			return;
		}
		if (!(rs instanceof RuleSymbol))
		{
			// Is this redundant???
			antlrTool.error("'" + rr.targetRule + "' does not name a grammar rule", grammar.getFilename(), rr.getLine(), rr.getColumn());
			return;
		}

		genErrorTryForElement(rr);

		// AST value for labeled rule refs in tree walker.
		// This is not AST construction;  it is just the input tree node value.
		if ( grammar instanceof TreeWalkerGrammar &&
			rr.getLabel() != null &&
			syntacticPredLevel == 0 )
		{
			println(rr.getLabel() + " = (_t == ASTNULL) ? "+labeledElementASTInit+" : "+lt1Value+";");
		}

		// if in lexer and ! on rule ref or alt or rule, save buffer index to
		// kill later
		if ( grammar instanceof LexerGrammar && (!saveText||rr.getAutoGenType()==GrammarElement.AUTO_GEN_BANG) )
		{
			println("_saveIndex = text.length();");
		}

		// Process return value assignment if any
		printTabs();
		if (rr.idAssign != null)
		{
			// Warn if the rule has no return type
			if (rs.block.returnAction == null)
			{
				antlrTool.warning("Rule '" + rr.targetRule + "' has no return type", grammar.getFilename(), rr.getLine(), rr.getColumn());
			}
			_print(rr.idAssign + "=");
		} else {
			// Warn about return value if any, but not inside syntactic predicate
			if ( !(grammar instanceof LexerGrammar) && syntacticPredLevel == 0 && rs.block.returnAction != null)
			{
				antlrTool.warning("Rule '" + rr.targetRule + "' returns a value", grammar.getFilename(), rr.getLine(), rr.getColumn());
			}
		}

		// Call the rule
		GenRuleInvocation(rr);

		// if in lexer and ! on element or alt or rule, save buffer index to kill later
		if ( grammar instanceof LexerGrammar && (!saveText||rr.getAutoGenType()==GrammarElement.AUTO_GEN_BANG) ) {
			println("text.erase(_saveIndex);");
		}

		// if not in a syntactic predicate
		if (syntacticPredLevel == 0)
		{
			boolean doNoGuessTest = (
				grammar.hasSyntacticPredicate &&
				(
					grammar.buildAST && rr.getLabel() != null ||
					(genAST && rr.getAutoGenType() == GrammarElement.AUTO_GEN_NONE)
				)
			);

			if (doNoGuessTest) {
				println("if (inputState->guessing==0) {");
				tabs++;
			}

			if (grammar.buildAST && rr.getLabel() != null)
			{
				// always gen variable for rule return on labeled rules
				// RK: hmm do I know here if the returnAST needs a cast ?
				println(rr.getLabel() + "_AST = returnAST;");
			}

			if (genAST)
			{
				switch (rr.getAutoGenType())
				{
				case GrammarElement.AUTO_GEN_NONE:
					if( usingCustomAST )
						println("astFactory->addASTChild(currentAST, "+namespaceAntlr+"RefAST(returnAST));");
					else
						println("astFactory->addASTChild( currentAST, returnAST );");
					break;
				case GrammarElement.AUTO_GEN_CARET:
					// FIXME: RK: I'm not so sure this should be an error..
					// I think it might actually work and be usefull at times.
					antlrTool.error("Internal: encountered ^ after rule reference");
					break;
				default:
					break;
				}
			}

			// if a lexer and labeled, Token label defined at rule level, just set it here
			if ( grammar instanceof LexerGrammar && rr.getLabel() != null )
			{
				println(rr.getLabel()+"=_returnToken;");
			}

			if (doNoGuessTest)
			{
				tabs--;
				println("}");
			}
		}
		genErrorCatchForElement(rr);
	}
	/** Generate code for the given grammar element.
	 * @param blk The string-literal reference to generate
	 */
	public void gen(StringLiteralElement atom) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("genString("+atom+")");

		// Variable declarations for labeled elements
		if (atom.getLabel()!=null && syntacticPredLevel == 0) {
			println(atom.getLabel() + " = " + lt1Value + ";");
		}

		// AST
		genElementAST(atom);

		// is there a bang on the literal?
		boolean oldsaveText = saveText;
		saveText = saveText && atom.getAutoGenType()==GrammarElement.AUTO_GEN_NONE;

		// matching
		genMatch(atom);

		saveText = oldsaveText;

		// tack on tree cursor motion if doing a tree walker
		if (grammar instanceof TreeWalkerGrammar) {
			println("_t = _t->getNextSibling();");
		}
	}
	/** Generate code for the given grammar element.
	 * @param blk The token-range reference to generate
	 */
	public void gen(TokenRangeElement r) {
		genErrorTryForElement(r);
		if ( r.getLabel()!=null  && syntacticPredLevel == 0) {
			println(r.getLabel() + " = " + lt1Value + ";");
		}

		// AST
		genElementAST(r);

		// match
		println("matchRange("+r.beginText+","+r.endText+");");
		genErrorCatchForElement(r);
	}
	/** Generate code for the given grammar element.
	 * @param blk The token-reference to generate
	 */
	public void gen(TokenRefElement atom) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("genTokenRef("+atom+")");
		if ( grammar instanceof LexerGrammar ) {
			antlrTool.panic("Token reference found in lexer");
		}
		genErrorTryForElement(atom);
		// Assign Token value to token label variable
		if ( atom.getLabel()!=null && syntacticPredLevel == 0) {
			println(atom.getLabel() + " = " + lt1Value + ";");
		}

		// AST
		genElementAST(atom);
		// matching
		genMatch(atom);
		genErrorCatchForElement(atom);

		// tack on tree cursor motion if doing a tree walker
		if (grammar instanceof TreeWalkerGrammar) {
			println("_t = _t->getNextSibling();");
		}
	}
	public void gen(TreeElement t) {
		// save AST cursor
		println(labeledElementType+" __t" + t.ID + " = _t;");

		// If there is a label on the root, then assign that to the variable
		if (t.root.getLabel() != null) {
			println(t.root.getLabel() + " = (_t == ASTNULL) ? "+labeledElementASTInit+" : _t;");
		}

		// check for invalid modifiers ! and ^ on tree element roots
		if ( t.root.getAutoGenType() == GrammarElement.AUTO_GEN_BANG ) {
			antlrTool.error("Suffixing a root node with '!' is not implemented",
						  grammar.getFilename(), t.getLine(), t.getColumn());
			t.root.setAutoGenType(GrammarElement.AUTO_GEN_NONE);
		}
		if ( t.root.getAutoGenType() == GrammarElement.AUTO_GEN_CARET ) {
			antlrTool.warning("Suffixing a root node with '^' is redundant; already a root",
							 grammar.getFilename(), t.getLine(), t.getColumn());
			t.root.setAutoGenType(GrammarElement.AUTO_GEN_NONE);
		}

		// Generate AST variables
		genElementAST(t.root);
		if (grammar.buildAST) {
			// Save the AST construction state
			println(namespaceAntlr+"ASTPair __currentAST" + t.ID + " = currentAST;");
			// Make the next item added a child of the TreeElement root
			println("currentAST.root = currentAST.child;");
			println("currentAST.child = "+labeledElementASTInit+";");
		}

		// match root
		if ( t.root instanceof WildcardElement ) {
			println("if ( _t == ASTNULL ) throw "+namespaceAntlr+"MismatchedTokenException();");
		}
		else {
			genMatch(t.root);
		}
		// move to list of children
		println("_t = _t->getFirstChild();");

		// walk list of children, generating code for each
		for (int i=0; i<t.getAlternatives().size(); i++) {
			Alternative a = t.getAlternativeAt(i);
			AlternativeElement e = a.head;
			while ( e != null ) {
				e.generate();
				e = e.next;
			}
		}

		if (grammar.buildAST) {
			// restore the AST construction state to that just after the
			// tree root was added
			println("currentAST = __currentAST" + t.ID + ";");
		}
		// restore AST cursor
		println("_t = __t" + t.ID + ";");
		// move cursor to sibling of tree just parsed
		println("_t = _t->getNextSibling();");
	}
	/** Generate the tree-parser C++ files */
	public void gen(TreeWalkerGrammar g) throws IOException {
		setGrammar(g);
		if (!(grammar instanceof TreeWalkerGrammar)) {
			antlrTool.panic("Internal error generating tree-walker");
		}

		genBody(g);
		genInclude(g);
	}
	/** Generate code for the given grammar element.
	 * @param wc The wildcard element to generate
	 */
	public void gen(WildcardElement wc) {
		// Variable assignment for labeled elements
		if (wc.getLabel()!=null && syntacticPredLevel == 0) {
			println(wc.getLabel() + " = " + lt1Value + ";");
		}

		// AST
		genElementAST(wc);
		// Match anything but EOF
		if (grammar instanceof TreeWalkerGrammar) {
			println("if ( _t == "+labeledElementASTInit+" ) throw "+namespaceAntlr+"MismatchedTokenException();");
		}
		else if (grammar instanceof LexerGrammar) {
			if ( grammar instanceof LexerGrammar &&
					(!saveText||wc.getAutoGenType()==GrammarElement.AUTO_GEN_BANG) ) {
				println("_saveIndex = text.length();");
			}
			println("matchNot(EOF/*_CHAR*/);");
			if ( grammar instanceof LexerGrammar &&
					(!saveText||wc.getAutoGenType()==GrammarElement.AUTO_GEN_BANG) ) {
				println("text.erase(_saveIndex);");      // kill text atom put in buffer
			}
		}
		else {
			println("matchNot(" + getValueString(Token.EOF_TYPE) + ");");
		}

		// tack on tree cursor motion if doing a tree walker
		if (grammar instanceof TreeWalkerGrammar) {
			println("_t = _t->getNextSibling();");
		}
	}
	/** Generate code for the given grammar element.
	 * @param blk The (...)* block to generate
	 */
	public void gen(ZeroOrMoreBlock blk) {
		if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) System.out.println("gen*("+blk+")");
		println("{ // ( ... )*");
		genBlockPreamble(blk);
		String label;
		if ( blk.getLabel() != null ) {
			label = blk.getLabel();
		}
		else {
			label = "_loop" + blk.ID;
		}
		println("for (;;) {");
		tabs++;
		// generate the init action for ()+ ()* inside the loop
		// this allows us to do usefull EOF checking...
		genBlockInitAction(blk);

		// Tell AST generation to build subrule result
		String saveCurrentASTResult = currentASTResult;
		if (blk.getLabel() != null) {
			currentASTResult = blk.getLabel();
		}

		boolean ok = grammar.theLLkAnalyzer.deterministic(blk);

		// generate exit test if greedy set to false
		// and an alt is ambiguous with exit branch
		// or when lookahead derived purely from end-of-file
		// Lookahead analysis stops when end-of-file is hit,
		// returning set {epsilon}.  Since {epsilon} is not
		// ambig with any real tokens, no error is reported
		// by deterministic() routines and we have to check
		// for the case where the lookahead depth didn't get
		// set to NONDETERMINISTIC (this only happens when the
		// FOLLOW contains real atoms + epsilon).
		boolean generateNonGreedyExitPath = false;
		int nonGreedyExitDepth = grammar.maxk;

		if ( !blk.greedy &&
			 blk.exitLookaheadDepth<=grammar.maxk &&
			 blk.exitCache[blk.exitLookaheadDepth].containsEpsilon() )
		{
			generateNonGreedyExitPath = true;
			nonGreedyExitDepth = blk.exitLookaheadDepth;
		}
		else if ( !blk.greedy &&
				  blk.exitLookaheadDepth==LLkGrammarAnalyzer.NONDETERMINISTIC )
		{
			generateNonGreedyExitPath = true;
		}
		if ( generateNonGreedyExitPath ) {
			if ( DEBUG_CODE_GENERATOR || DEBUG_CPP_CODE_GENERATOR ) {
				System.out.println("nongreedy (...)* loop; exit depth is "+
								   blk.exitLookaheadDepth);
			}
			String predictExit =
				getLookaheadTestExpression(blk.exitCache,
										   nonGreedyExitDepth);
			println("// nongreedy exit test");
			println("if ("+predictExit+") goto "+label+";");
		}

		CppBlockFinishingInfo howToFinish = genCommonBlock(blk, false);
		genBlockFinish(howToFinish, "goto " + label + ";");

		tabs--;
		println("}");
		println(label+":;");
		println("} // ( ... )*");

		// Restore previous AST generation
		currentASTResult = saveCurrentASTResult;
	}
	/** Generate an alternative.
	  * @param alt  The alternative to generate
	  * @param blk The block to which the alternative belongs
	  */
	protected void genAlt(Alternative alt, AlternativeBlock blk)
	{
		// Save the AST generation state, and set it to that of the alt
		boolean savegenAST = genAST;
		genAST = genAST && alt.getAutoGen();

		boolean oldsaveTest = saveText;
		saveText = saveText && alt.getAutoGen();

		// Reset the variable name map for the alternative
		Hashtable saveMap = treeVariableMap;
		treeVariableMap = new Hashtable();

		// Generate try block around the alt for  error handling
		if (alt.exceptionSpec != null) {
			println("try {      // for error handling");
			tabs++;
		}

		AlternativeElement elem = alt.head;
		while ( !(elem instanceof BlockEndElement) ) {
			elem.generate(); // alt can begin with anything. Ask target to gen.
			elem = elem.next;
		}

		if ( genAST)
		{
			if (blk instanceof RuleBlock)
			{
				// Set the AST return value for the rule
				RuleBlock rblk = (RuleBlock)blk;
				if( usingCustomAST )
					println(rblk.getRuleName() + "_AST = "+labeledElementASTType+"(currentAST.root);");
				else
					println(rblk.getRuleName() + "_AST = currentAST.root;");
			}
			else if (blk.getLabel() != null) {
				// ### future: also set AST value for labeled subrules.
				// println(blk.getLabel() + "_AST = "+labeledElementASTType+"(currentAST.root);");
				antlrTool.warning("Labeled subrules are not implemented", grammar.getFilename(), blk.getLine(), blk.getColumn());
			}
		}

		if (alt.exceptionSpec != null)
		{
			// close try block
			tabs--;
			println("}");
			genErrorHandler(alt.exceptionSpec);
		}

		genAST = savegenAST;
		saveText = oldsaveTest;

		treeVariableMap = saveMap;
	}
	/** Generate all the bitsets to be used in the parser or lexer
	 * Generate the raw bitset data like "long _tokenSet1_data[] = {...};"
	 * and the BitSet object declarations like
	 * "BitSet _tokenSet1 = new BitSet(_tokenSet1_data);"
	 * Note that most languages do not support object initialization inside a
	 * class definition, so other code-generators may have to separate the
	 * bitset declarations from the initializations (e.g., put the
	 * initializations in the generated constructor instead).
	 * @param bitsetList The list of bitsets to generate.
	 * @param maxVocabulary Ensure that each generated bitset can contain at
	 *        least this value.
	 * @param prefix string glued in from of bitset names used for namespace
	 *        qualifications.
	 */
	protected void genBitsets(
		Vector bitsetList,
		int maxVocabulary,
		String prefix
	)
	{
		TokenManager tm = grammar.tokenManager;

		println("");

		for (int i = 0; i < bitsetList.size(); i++)
		{
			BitSet p = (BitSet)bitsetList.elementAt(i);
			// Ensure that generated BitSet is large enough for vocabulary
			p.growToInclude(maxVocabulary);

			// initialization data
			println(
				"const unsigned long " + prefix + getBitsetName(i) + "_data_" + "[] = { " +
				p.toStringOfHalfWords() +
				" };"
			);

			// Dump the contents of the bitset in readable format...
			String t = "// ";

			for( int j = 0; j < tm.getVocabulary().size(); j++ )
			{
				if ( p.member( j ) )
				{
					if ( (grammar instanceof LexerGrammar) )
					{
						// only dump out for pure printable ascii.
						if( ( 0x20 <= j ) && ( j < 0x7F ) )
							t += charFormatter.escapeChar(j,true)+" ";
						else
							t += "0x"+Integer.toString(j,16)+" ";
					}
					else
						t += tm.getTokenStringAt(j)+" ";

					if( t.length() > 70 )
					{
						println(t);
						t = "// ";
					}
				}
			}
			if ( t != "// " )
				println(t);

			// BitSet object
			println(
				"const "+namespaceAntlr+"BitSet " + prefix + getBitsetName(i) + "(" +
				getBitsetName(i) + "_data_," + p.size()/32 +
				");"
			);
		}
	}
	protected void genBitsetsHeader(
		Vector bitsetList,
		int maxVocabulary
	) {
		println("");
		for (int i = 0; i < bitsetList.size(); i++)
		{
			BitSet p = (BitSet)bitsetList.elementAt(i);
			// Ensure that generated BitSet is large enough for vocabulary
			p.growToInclude(maxVocabulary);
			// initialization data
			println("static const unsigned long " + getBitsetName(i) + "_data_" + "[];");
			// BitSet object
			println("static const "+namespaceAntlr+"BitSet " + getBitsetName(i) + ";");
		}
	}
	/** Generate the finish of a block, using a combination of the info
	 * returned from genCommonBlock() and the action to perform when
	 * no alts were taken
	 * @param howToFinish The return of genCommonBlock()
	 * @param noViableAction What to generate when no alt is taken
	 */
	private void genBlockFinish(CppBlockFinishingInfo howToFinish, String noViableAction)
	{
		if (howToFinish.needAnErrorClause &&
			 (howToFinish.generatedAnIf || howToFinish.generatedSwitch)) {
			if ( howToFinish.generatedAnIf ) {
				println("else {");
			}
			else {
				println("{");
			}
			tabs++;
			println(noViableAction);
			tabs--;
			println("}");
		}

		if ( howToFinish.postscript!=null ) {
			println(howToFinish.postscript);
		}
	}
	/** Generate the initaction for a block, which may be a RuleBlock or a
	 * plain AlternativeBLock.
	 * @blk The block for which the preamble is to be generated.
	 */
	protected void genBlockInitAction( AlternativeBlock blk )
	{
		// dump out init action
		if ( blk.initAction!=null ) {
			genLineNo(blk);
			printAction(processActionForSpecialSymbols(blk.initAction, blk.line,
																	 currentRule, null) );
			genLineNo2();
		}
	}
	/** Generate the header for a block, which may be a RuleBlock or a
	 * plain AlternativeBlock. This generates any variable declarations
	 * and syntactic-predicate-testing variables.
	 * @blk The block for which the preamble is to be generated.
	 */
	protected void genBlockPreamble(AlternativeBlock blk) {
		// define labels for rule blocks.
		if ( blk instanceof RuleBlock ) {
			RuleBlock rblk = (RuleBlock)blk;
			if ( rblk.labeledElements!=null ) {
				for (int i=0; i<rblk.labeledElements.size(); i++) {

					AlternativeElement a = (AlternativeElement)rblk.labeledElements.elementAt(i);
					//System.out.println("looking at labeled element: "+a);
					// Variables for labeled rule refs and subrules are different than
					// variables for grammar atoms.  This test is a little tricky because
					// we want to get all rule refs and ebnf, but not rule blocks or
					// syntactic predicates
					if (
						a instanceof RuleRefElement ||
						a instanceof AlternativeBlock &&
						!(a instanceof RuleBlock) &&
						!(a instanceof SynPredBlock) )
					{
						if ( !(a instanceof RuleRefElement) &&
							  ((AlternativeBlock)a).not &&
							  analyzer.subruleCanBeInverted(((AlternativeBlock)a), grammar instanceof LexerGrammar)
						) {
							// Special case for inverted subrules that will be
							// inlined. Treat these like token or char literal
							// references
							println(labeledElementType + " " + a.getLabel() + " = " + labeledElementInit + ";");
							if (grammar.buildAST) {
								genASTDeclaration( a );
							}
						}
						else
						{
							if (grammar.buildAST)
							{
								// Always gen AST variables for labeled elements,
								// even if the element itself is marked with !
								genASTDeclaration( a );
							}
							if ( grammar instanceof LexerGrammar )
								println(namespaceAntlr+"RefToken "+a.getLabel()+";");

							if (grammar instanceof TreeWalkerGrammar) {
								// always generate rule-ref variables for tree walker
								println(labeledElementType + " " + a.getLabel() + " = " + labeledElementInit + ";");
							}
						}
					}
					else
					{
						// It is a token or literal reference.  Generate the
						// correct variable type for this grammar
						println(labeledElementType + " " + a.getLabel() + " = " + labeledElementInit + ";");
						// In addition, generate *_AST variables if building ASTs
						if (grammar.buildAST)
						{
							if (a instanceof GrammarAtom &&
								 ((GrammarAtom)a).getASTNodeType() != null )
							{
								GrammarAtom ga = (GrammarAtom)a;
								genASTDeclaration( a, "Ref"+ga.getASTNodeType() );
							}
							else
							{
								genASTDeclaration( a );
							}
						}
					}
				}
			}
		}
	}
	public void genBody(LexerGrammar g) throws IOException
	{
		outputFile = grammar.getClassName() + ".cpp";
		outputLine = 1;
		currentOutput = antlrTool.openOutputFile(outputFile);
		//SAS: changed for proper text file io

		genAST = false;	// no way to gen trees.
		saveText = true;	// save consumed characters.

		tabs=0;

		// Generate header common to all C++ output files
		genHeader(outputFile);

		printHeaderAction(preIncludeCpp);
		// Generate header specific to lexer C++ file
		println("#include \"" + grammar.getClassName() + ".hpp\"");
		println("#include <antlr/CharBuffer.hpp>");
		println("#include <antlr/TokenStreamException.hpp>");
		println("#include <antlr/TokenStreamIOException.hpp>");
		println("#include <antlr/TokenStreamRecognitionException.hpp>");
		println("#include <antlr/CharStreamException.hpp>");
		println("#include <antlr/CharStreamIOException.hpp>");
		println("#include <antlr/NoViableAltForCharException.hpp>");
		if (grammar.debuggingOutput)
			println("#include <antlr/DebuggingInputBuffer.hpp>");
		println("");
		printHeaderAction(postIncludeCpp);

		if (nameSpace != null)
			nameSpace.emitDeclarations(currentOutput);

		// Generate user-defined lexer file preamble
		printAction(grammar.preambleAction);

		// Generate lexer class definition
		String sup=null;
		if ( grammar.superClass!=null ) {
			sup = grammar.superClass;
		}
		else {
			sup = grammar.getSuperClass();
			if (sup.lastIndexOf('.') != -1)
				sup = sup.substring(sup.lastIndexOf('.')+1);
			sup = namespaceAntlr + sup;
		}

		if( noConstructors )
		{
			println("#if 0");
			println("// constructor creation turned of with 'noConstructor' option");
		}
		//
		// Generate the constructor from InputStream
		//
		println(grammar.getClassName() + "::" + grammar.getClassName() + "(" + namespaceStd + "istream& in)");
		tabs++;
		// if debugging, wrap the input buffer in a debugger
		if (grammar.debuggingOutput)
			println(": " + sup + "(new "+namespaceAntlr+"DebuggingInputBuffer(new "+namespaceAntlr+"CharBuffer(in)),"+g.caseSensitive+")");
		else
			println(": " + sup + "(new "+namespaceAntlr+"CharBuffer(in),"+g.caseSensitive+")");
		tabs--;
		println("{");
		tabs++;

		// if debugging, set up array variables and call user-overridable
		//   debugging setup method
		if ( grammar.debuggingOutput ) {
			println("setRuleNames(_ruleNames);");
			println("setSemPredNames(_semPredNames);");
			println("setupDebugging();");
		}

//		println("setCaseSensitive("+g.caseSensitive+");");
		println("initLiterals();");
		tabs--;
		println("}");
		println("");

		// Generate the constructor from InputBuffer
		println(grammar.getClassName() + "::" + grammar.getClassName() + "("+namespaceAntlr+"InputBuffer& ib)");
		tabs++;
		// if debugging, wrap the input buffer in a debugger
		if (grammar.debuggingOutput)
			println(": " + sup + "(new "+namespaceAntlr+"DebuggingInputBuffer(ib),"+g.caseSensitive+")");
		else
			println(": " + sup + "(ib,"+g.caseSensitive+")");
		tabs--;
		println("{");
		tabs++;

		// if debugging, set up array variables and call user-overridable
		//   debugging setup method
		if ( grammar.debuggingOutput ) {
			println("setRuleNames(_ruleNames);");
			println("setSemPredNames(_semPredNames);");
			println("setupDebugging();");
		}

//		println("setCaseSensitive("+g.caseSensitive+");");
		println("initLiterals();");
		tabs--;
		println("}");
		println("");

		// Generate the constructor from LexerSharedInputState
		println(grammar.getClassName() + "::" + grammar.getClassName() + "(const "+namespaceAntlr+"LexerSharedInputState& state)");
		tabs++;
		println(": " + sup + "(state,"+g.caseSensitive+")");
		tabs--;
		println("{");
		tabs++;

		// if debugging, set up array variables and call user-overridable
		//   debugging setup method
		if ( grammar.debuggingOutput ) {
			println("setRuleNames(_ruleNames);");
			println("setSemPredNames(_semPredNames);");
			println("setupDebugging();");
		}

//		println("setCaseSensitive("+g.caseSensitive+");");
		println("initLiterals();");
		tabs--;
		println("}");
		println("");

		if( noConstructors )
		{
			println("// constructor creation turned of with 'noConstructor' option");
			println("#endif");
		}

		println("void " + grammar.getClassName() + "::initLiterals()");
		println("{");
		tabs++;
		// Generate the initialization of the map
		// containing the string literals used in the lexer
		// The literals variable itself is in CharScanner
		Enumeration keys = grammar.tokenManager.getTokenSymbolKeys();
		while ( keys.hasMoreElements() ) {
			String key = (String)keys.nextElement();
			if ( key.charAt(0) != '"' ) {
				continue;
			}
			TokenSymbol sym = grammar.tokenManager.getTokenSymbol(key);
			if ( sym instanceof StringLiteralSymbol ) {
				StringLiteralSymbol s = (StringLiteralSymbol)sym;
				println("literals["+s.getId()+"] = "+s.getTokenType()+";");
			}
		}

		// Generate the setting of various generated options.
		tabs--;
		println("}");

		Enumeration ids;
		// generate the rule name array for debugging
		if (grammar.debuggingOutput) {
			println("const char* "+grammar.getClassName()+"::_ruleNames[] = {");
			tabs++;

			ids = grammar.rules.elements();
			int ruleNum=0;
			while ( ids.hasMoreElements() ) {
				GrammarSymbol sym = (GrammarSymbol) ids.nextElement();
				if ( sym instanceof RuleSymbol)
					println("\""+((RuleSymbol)sym).getId()+"\",");
			}
			println("0");
			tabs--;
			println("};");
		}

		// Generate nextToken() rule.
		// nextToken() is a synthetic lexer rule that is the implicit OR of all
		// user-defined lexer rules.
		genNextToken();

		// Generate code for each rule in the lexer
		ids = grammar.rules.elements();
		int ruleNum=0;
		while ( ids.hasMoreElements() ) {
			RuleSymbol sym = (RuleSymbol) ids.nextElement();
			// Don't generate the synthetic rules
			if (!sym.getId().equals("mnextToken")) {
				genRule(sym, false, ruleNum++, grammar.getClassName() + "::");
			}
			exitIfError();
		}

		// Generate the semantic predicate map for debugging
		if (grammar.debuggingOutput)
			genSemPredMap(grammar.getClassName() + "::");

		// Generate the bitsets used throughout the lexer
		genBitsets(bitsetsUsed, ((LexerGrammar)grammar).charVocabulary.size(), grammar.getClassName() + "::" );

		println("");
		if (nameSpace != null)
			nameSpace.emitClosures(currentOutput);

		// Close the lexer output stream
		currentOutput.close();
		currentOutput = null;
	}
	public void genInitFactory( Grammar g )
	{
		// Generate the method to initialize an ASTFactory when we're
		// building AST's
		String param_name = "factory ";
		if( ! g.buildAST )
			param_name = "";

		println("void "+ g.getClassName() + "::initializeASTFactory( "+namespaceAntlr+"ASTFactory& "+param_name+")");
		println("{");
		tabs++;

		if( g.buildAST )
		{
			// sort out custom AST types... synchronize token manager with token
			// specs on rules (and other stuff we were able to see from
			// action.g) (imperfect of course)
			TokenManager tm = grammar.tokenManager;
			Enumeration tokens = tm.getTokenSymbolKeys();
			while( tokens.hasMoreElements() )
			{
				String tok = (String)tokens.nextElement();
				TokenSymbol ts = tm.getTokenSymbol(tok);
				// if we have a custom type and there's not a more local override
				// of the tokentype then mark this as the type for the tokentype
				if( ts.getASTNodeType() != null )
				{
					// ensure capacity with this pseudo vector...
					astTypes.ensureCapacity(ts.getTokenType());
					String type = (String)astTypes.elementAt(ts.getTokenType());
					if( type == null )
						astTypes.setElementAt(ts.getASTNodeType(),ts.getTokenType());
					else
					{
						// give a warning over action taken if the types are unequal
						if( ! ts.getASTNodeType().equals(type) )
						{
							antlrTool.warning("Token "+tok+" taking most specific AST type",grammar.getFilename(),1,1);
							antlrTool.warning("  using "+type+" ignoring "+ts.getASTNod…