PageRenderTime 840ms CodeModel.GetById 201ms app.highlight 467ms RepoModel.GetById 157ms app.codeStats 1ms

/tool/src/main/java/org/antlr/Tool.java

https://bitbucket.org/nickolas_pohilets/antlr3
Java | 1429 lines | 1084 code | 62 blank | 283 comment | 46 complexity | e7eafe0ada82e44959f0dcddc805e806 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/*
   2 * [The "BSD license"]
   3 *  Copyright (c) 2010 Terence Parr
   4 *  All rights reserved.
   5 *
   6 *  Redistribution and use in source and binary forms, with or without
   7 *  modification, are permitted provided that the following conditions
   8 *  are met:
   9 *  1. Redistributions of source code must retain the above copyright
  10 *      notice, this list of conditions and the following disclaimer.
  11 *  2. Redistributions in binary form must reproduce the above copyright
  12 *      notice, this list of conditions and the following disclaimer in the
  13 *      documentation and/or other materials provided with the distribution.
  14 *  3. The name of the author may not be used to endorse or promote products
  15 *      derived from this software without specific prior written permission.
  16 *
  17 *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18 *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19 *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20 *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22 *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26 *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27 */
  28package org.antlr;
  29
  30import org.antlr.analysis.*;
  31import org.antlr.codegen.CodeGenerator;
  32import org.antlr.misc.Graph;
  33import org.antlr.runtime.misc.Stats;
  34import org.antlr.tool.*;
  35import org.stringtemplate.v4.STGroup;
  36
  37import java.io.*;
  38import java.util.*;
  39
  40/** The main ANTLR entry point.  Read a grammar and generate a parser. */
  41public class Tool {
  42
  43    public final Properties antlrSettings = new Properties();
  44
  45	public final String VERSION;
  46	{
  47		String version = Tool.class.getPackage().getImplementationVersion();
  48		VERSION = version != null ? version : "3.x";
  49	}
  50
  51    public static final String UNINITIALIZED_DIR = "<unset-dir>";
  52    private List<String> grammarFileNames = new ArrayList<String>();
  53    private boolean generate_NFA_dot = false;
  54    private boolean generate_DFA_dot = false;
  55    private String outputDirectory = ".";
  56    private boolean haveOutputDir = false;
  57    private String inputDirectory = null;
  58    private String parentGrammarDirectory;
  59    private String grammarOutputDirectory;
  60    private boolean haveInputDir = false;
  61    private String libDirectory = ".";
  62    private boolean debug = false;
  63    private boolean trace = false;
  64    private boolean profile = false;
  65    private boolean report = false;
  66    private boolean printGrammar = false;
  67    private boolean depend = false;
  68    private boolean forceAllFilesToOutputDir = false;
  69    private boolean forceRelativeOutput = false;
  70    protected boolean deleteTempLexer = true;
  71    private boolean verbose = false;
  72    /** Don't process grammar file if generated files are newer than grammar */
  73    private boolean make = false;
  74    private boolean showBanner = true;
  75	private static boolean exitNow = false;
  76	private static boolean return_dont_exit = false;
  77
  78
  79	public String forcedLanguageOption; // -language L on command line
  80
  81    // The internal options are for my use on the command line during dev
  82    //
  83    public static boolean internalOption_PrintGrammarTree = false;
  84    public static boolean internalOption_PrintDFA = false;
  85    public static boolean internalOption_ShowNFAConfigsInDFA = false;
  86    public static boolean internalOption_watchNFAConversion = false;
  87
  88    /**
  89     * A list of dependency generators that are accumulated aaaas (and if) the
  90     * tool is required to sort the provided grammars into build dependency order.
  91    protected Map<String, BuildDependencyGenerator> buildDependencyGenerators;
  92     */
  93
  94    public static void main(String[] args) {
  95        Tool antlr = new Tool(args);
  96
  97        if (!exitNow) {
  98            antlr.process();
  99			if ( return_dont_exit ) return;
 100            if (ErrorManager.getNumErrors() > 0) {
 101                System.exit(1);
 102            }
 103            System.exit(0);
 104        }
 105    }
 106
 107    /**
 108     * Load the properties file org/antlr/antlr.properties and populate any
 109     * variables that must be initialized from it, such as the version of ANTLR.
 110     */
 111    private void loadResources() {
 112        InputStream in;
 113        in = this.getClass().getResourceAsStream("antlr.properties");
 114
 115        // If we found the resource, then load it, otherwise revert to the
 116        // defaults.
 117        //
 118        if (in != null) {
 119            try {
 120                // Load the resources into the map
 121                //
 122                antlrSettings.load(in);
 123
 124                // Set any variables that we need to populate from the resources
 125                //
 126//                VERSION = antlrSettings.getProperty("antlr.version");
 127            } catch (Exception e) {
 128                // Do nothing, just leave the defaults in place
 129            }
 130        }
 131    }
 132
 133    public Tool() {
 134        loadResources();
 135    }
 136
 137	@SuppressWarnings("OverridableMethodCallInConstructor")
 138    public Tool(String[] args) {
 139        loadResources();
 140
 141        // Set all the options and pick up all the named grammar files
 142        processArgs(args);
 143    }
 144
 145    public void processArgs(String[] args) {
 146
 147        if (isVerbose()) {
 148            ErrorManager.info("ANTLR Parser Generator  Version " + VERSION);
 149            showBanner = false;
 150        }
 151
 152        if (args == null || args.length == 0) {
 153            help();
 154            return;
 155        }
 156        for (int i = 0; i < args.length; i++) {
 157            if (args[i].equals("-o") || args[i].equals("-fo") || args[i].equals("-ro")) {
 158                if (i + 1 >= args.length) {
 159                    System.err.println("missing output directory with " + args[i] + " option; ignoring");
 160                }
 161                else {
 162                    if (args[i].equals("-fo")) { // force output into dir
 163                        setForceAllFilesToOutputDir(true);
 164                    }
 165					else if (args[i].equals("-ro")) {
 166						setForceRelativeOutput(true);
 167					}
 168                    i++;
 169                    outputDirectory = args[i];
 170                    if (outputDirectory.endsWith("/") ||
 171                        outputDirectory.endsWith("\\")) {
 172                        outputDirectory =
 173                            outputDirectory.substring(0, getOutputDirectory().length() - 1);
 174                    }
 175                    File outDir = new File(outputDirectory);
 176                    haveOutputDir = true;
 177                    if (outDir.exists() && !outDir.isDirectory()) {
 178                        ErrorManager.error(ErrorManager.MSG_OUTPUT_DIR_IS_FILE, outputDirectory);
 179                        setLibDirectory(".");
 180                    }
 181                }
 182            }
 183			else if (args[i].equals("-in")) {
 184				if (i + 1 >= args.length) {
 185					System.err.println("missing input directory with -in option; ignoring");
 186				}
 187				else {
 188					i++;
 189					String directory = args[i];
 190					if (directory.endsWith("/") || directory.endsWith("\\")) {
 191						directory = directory.substring(0, directory.length() - 1);
 192					}
 193
 194					if (!new File(directory).isDirectory()) {
 195						System.err.println("the specified input directory does not exist");
 196						continue;
 197					}
 198
 199					setInputDirectory(args[i]);
 200				}
 201			}
 202			else if (args[i].equals("-lib")) {
 203				if (i + 1 >= args.length) {
 204					System.err.println("missing library directory with -lib option; ignoring");
 205				}
 206				else {
 207					i++;
 208					setLibDirectory(args[i]);
 209					if (getLibraryDirectory().endsWith("/") ||
 210						getLibraryDirectory().endsWith("\\")) {
 211						setLibDirectory(getLibraryDirectory().substring(0, getLibraryDirectory().length() - 1));
 212					}
 213					File outDir = new File(getLibraryDirectory());
 214					if (!outDir.exists()) {
 215						ErrorManager.error(ErrorManager.MSG_DIR_NOT_FOUND, getLibraryDirectory());
 216						setLibDirectory(".");
 217					}
 218				}
 219			}
 220			else if (args[i].equals("-language")) {
 221				if (i + 1 >= args.length) {
 222					System.err.println("missing language name; ignoring");
 223				}
 224				else {
 225					i++;
 226					forcedLanguageOption = args[i];
 227				}
 228			}
 229            else if (args[i].equals("-nfa")) {
 230                setGenerate_NFA_dot(true);
 231            }
 232            else if (args[i].equals("-dfa")) {
 233                setGenerate_DFA_dot(true);
 234            }
 235            else if (args[i].equals("-debug")) {
 236                setDebug(true);
 237            }
 238            else if (args[i].equals("-trace")) {
 239                setTrace(true);
 240            }
 241            else if (args[i].equals("-report")) {
 242                setReport(true);
 243            }
 244            else if (args[i].equals("-profile")) {
 245                setProfile(true);
 246            }
 247            else if (args[i].equals("-print")) {
 248                setPrintGrammar(true);
 249            }
 250            else if (args[i].equals("-depend")) {
 251                setDepend(true);
 252            }
 253            else if (args[i].equals("-verbose")) {
 254                setVerbose(true);
 255            }
 256            else if (args[i].equals("-version")) {
 257                version();
 258                exitNow = true;
 259            }
 260            else if (args[i].equals("-make")) {
 261                setMake(true);
 262            }
 263            else if (args[i].equals("-message-format")) {
 264                if (i + 1 >= args.length) {
 265                    System.err.println("missing output format with -message-format option; using default");
 266                }
 267                else {
 268                    i++;
 269                    ErrorManager.setFormat(args[i]);
 270                }
 271            }
 272            else if (args[i].equals("-Xgrtree")) {
 273                internalOption_PrintGrammarTree = true; // print grammar tree
 274            }
 275            else if (args[i].equals("-Xdfa")) {
 276                internalOption_PrintDFA = true;
 277            }
 278            else if (args[i].equals("-Xnoprune")) {
 279                DFAOptimizer.PRUNE_EBNF_EXIT_BRANCHES = false;
 280            }
 281            else if (args[i].equals("-Xnocollapse")) {
 282                DFAOptimizer.COLLAPSE_ALL_PARALLEL_EDGES = false;
 283            }
 284            else if (args[i].equals("-Xdbgconversion")) {
 285                NFAToDFAConverter.debug = true;
 286            }
 287            else if (args[i].equals("-Xmultithreaded")) {
 288                NFAToDFAConverter.SINGLE_THREADED_NFA_CONVERSION = false;
 289            }
 290            else if (args[i].equals("-Xnomergestopstates")) {
 291                DFAOptimizer.MERGE_STOP_STATES = false;
 292            }
 293            else if (args[i].equals("-Xdfaverbose")) {
 294                internalOption_ShowNFAConfigsInDFA = true;
 295            }
 296            else if (args[i].equals("-Xwatchconversion")) {
 297                internalOption_watchNFAConversion = true;
 298            }
 299            else if (args[i].equals("-XdbgST")) {
 300                CodeGenerator.LAUNCH_ST_INSPECTOR = true;
 301				STGroup.trackCreationEvents = true;
 302				return_dont_exit = true;
 303            }
 304            else if (args[i].equals("-Xmaxinlinedfastates")) {
 305                if (i + 1 >= args.length) {
 306                    System.err.println("missing max inline dfa states -Xmaxinlinedfastates option; ignoring");
 307                }
 308                else {
 309                    i++;
 310                    CodeGenerator.MAX_ACYCLIC_DFA_STATES_INLINE = Integer.parseInt(args[i]);
 311                }
 312            }
 313            else if (args[i].equals("-Xmaxswitchcaselabels")) {
 314                if (i + 1 >= args.length) {
 315                    System.err.println("missing max switch case labels -Xmaxswitchcaselabels option; ignoring");
 316                }
 317                else {
 318                    i++;
 319                    CodeGenerator.MAX_SWITCH_CASE_LABELS = Integer.parseInt(args[i]);
 320                }
 321            }
 322            else if (args[i].equals("-Xminswitchalts")) {
 323                if (i + 1 >= args.length) {
 324                    System.err.println("missing min switch alternatives -Xminswitchalts option; ignoring");
 325                }
 326                else {
 327                    i++;
 328                    CodeGenerator.MIN_SWITCH_ALTS = Integer.parseInt(args[i]);
 329                }
 330            }
 331            else if (args[i].equals("-Xm")) {
 332                if (i + 1 >= args.length) {
 333                    System.err.println("missing max recursion with -Xm option; ignoring");
 334                }
 335                else {
 336                    i++;
 337                    NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK = Integer.parseInt(args[i]);
 338                }
 339            }
 340            else if (args[i].equals("-Xmaxdfaedges")) {
 341                if (i + 1 >= args.length) {
 342                    System.err.println("missing max number of edges with -Xmaxdfaedges option; ignoring");
 343                }
 344                else {
 345                    i++;
 346                    DFA.MAX_STATE_TRANSITIONS_FOR_TABLE = Integer.parseInt(args[i]);
 347                }
 348            }
 349            else if (args[i].equals("-Xconversiontimeout")) {
 350                if (i + 1 >= args.length) {
 351                    System.err.println("missing max time in ms -Xconversiontimeout option; ignoring");
 352                }
 353                else {
 354                    i++;
 355                    DFA.MAX_TIME_PER_DFA_CREATION = Integer.parseInt(args[i]);
 356                }
 357            }
 358			else if (args[i].equals("-Xnfastates")) {
 359				DecisionProbe.verbose = true;
 360			}
 361			else if (args[i].equals("-Xsavelexer")) {
 362				deleteTempLexer = false;
 363			}
 364            else if (args[i].equals("-X")) {
 365                Xhelp();
 366            }
 367            else {
 368                if (args[i].charAt(0) != '-') {
 369                    // Must be the grammar file
 370                    addGrammarFile(args[i]);
 371                }
 372            }
 373        }
 374    }
 375
 376    /*
 377    protected void checkForInvalidArguments(String[] args, BitSet cmdLineArgValid) {
 378    // check for invalid command line args
 379    for (int a = 0; a < args.length; a++) {
 380    if (!cmdLineArgValid.member(a)) {
 381    System.err.println("invalid command-line argument: " + args[a] + "; ignored");
 382    }
 383    }
 384    }
 385     */
 386
 387    /**
 388     * Checks to see if the list of outputFiles all exist, and have
 389     * last-modified timestamps which are later than the last-modified
 390     * timestamp of all the grammar files involved in build the output
 391     * (imports must be checked). If these conditions hold, the method
 392     * returns false, otherwise, it returns true.
 393     *
 394     * @param grammarFileName The grammar file we are checking
 395     */
 396    public boolean buildRequired(String grammarFileName)
 397        throws IOException
 398    {
 399        BuildDependencyGenerator bd =
 400            new BuildDependencyGenerator(this, grammarFileName);
 401
 402        List<File> outputFiles = bd.getGeneratedFileList();
 403        List<File> inputFiles = bd.getDependenciesFileList();
 404        // Note that input directory must be set to use buildRequired
 405        File grammarFile;
 406        if (haveInputDir) {
 407            grammarFile = new File(inputDirectory, grammarFileName);
 408        }
 409        else {
 410            grammarFile = new File(grammarFileName);
 411        }
 412        long grammarLastModified = grammarFile.lastModified();
 413        for (File outputFile : outputFiles) {
 414            if (!outputFile.exists() || grammarLastModified > outputFile.lastModified()) {
 415                // One of the output files does not exist or is out of date, so we must build it
 416				if (isVerbose()) {
 417					if (!outputFile.exists()) {
 418						System.out.println("Output file " + outputFile + " does not exist: must build " + grammarFile);
 419					}
 420					else {
 421						System.out.println("Output file " + outputFile + " is not up-to-date: must build " + grammarFile);
 422					}
 423				}
 424
 425                return true;
 426            }
 427            // Check all of the imported grammars and see if any of these are younger
 428            // than any of the output files.
 429            if (inputFiles != null) {
 430                for (File inputFile : inputFiles) {
 431
 432                    if (inputFile.lastModified() > outputFile.lastModified()) {
 433                        // One of the imported grammar files has been updated so we must build
 434						if (isVerbose()) {
 435							System.out.println("Input file " + inputFile + " is newer than output: must rebuild " + grammarFile);
 436						}
 437
 438                        return true;
 439                    }
 440                }
 441            }
 442        }
 443        if (isVerbose()) {
 444            System.out.println("Grammar " + grammarFile + " is up to date - build skipped");
 445        }
 446        return false;
 447    }
 448
 449    public void process() {
 450        boolean exceptionWhenWritingLexerFile = false;
 451        String lexerGrammarFileName;		// necessary at this scope to have access in the catch below
 452
 453        // Have to be tricky here when Maven or build tools call in and must new Tool()
 454        // before setting options. The banner won't display that way!
 455        if (isVerbose() && showBanner) {
 456            ErrorManager.info("ANTLR Parser Generator  Version " + VERSION);
 457            showBanner = false;
 458        }
 459
 460        try {
 461            sortGrammarFiles(); // update grammarFileNames
 462        }
 463        catch (Exception e) {
 464            ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR,e);
 465        }
 466        catch (Error e) {
 467            ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
 468        }
 469
 470        for (String grammarFileName : grammarFileNames) {
 471            // If we are in make mode (to support build tools like Maven) and the
 472            // file is already up to date, then we do not build it (and in verbose mode
 473            // we will say so).
 474            if (make) {
 475                try {
 476                    if ( !buildRequired(grammarFileName) ) continue;
 477                }
 478                catch (Exception e) {
 479                    ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR,e);
 480                }
 481            }
 482
 483            if (isVerbose() && !isDepend()) {
 484                System.out.println(grammarFileName);
 485            }
 486            try {
 487                if (isDepend()) {
 488                    BuildDependencyGenerator dep =
 489                        new BuildDependencyGenerator(this, grammarFileName);
 490                    /*
 491                    List outputFiles = dep.getGeneratedFileList();
 492                    List dependents = dep.getDependenciesFileList();
 493                    System.out.println("output: "+outputFiles);
 494                    System.out.println("dependents: "+dependents);
 495                     */
 496                    System.out.println(dep.getDependencies().render());
 497                    continue;
 498                }
 499
 500                Grammar rootGrammar = getRootGrammar(grammarFileName);
 501                // we now have all grammars read in as ASTs
 502                // (i.e., root and all delegates)
 503				rootGrammar.composite.assignTokenTypes();
 504				//rootGrammar.composite.translateLeftRecursiveRules();
 505				rootGrammar.addRulesForSyntacticPredicates();
 506				rootGrammar.composite.defineGrammarSymbols();
 507                rootGrammar.composite.createNFAs();
 508
 509                generateRecognizer(rootGrammar);
 510
 511                if (isPrintGrammar()) {
 512                    rootGrammar.printGrammar(System.out);
 513                }
 514
 515                if (isReport()) {
 516					GrammarReport2 greport = new GrammarReport2(rootGrammar);
 517					System.out.print(greport.toString());
 518//                    GrammarReport greport = new GrammarReport(rootGrammar);
 519//                    System.out.println(greport.toString());
 520//                    // print out a backtracking report too (that is not encoded into log)
 521//                    System.out.println(greport.getBacktrackingReport());
 522                }
 523                if (isProfile()) {
 524                    GrammarReport greport = new GrammarReport(rootGrammar);
 525                    Stats.writeReport(GrammarReport.GRAMMAR_STATS_FILENAME,
 526                                      greport.toNotifyString());
 527                }
 528
 529                // now handle the lexer if one was created for a merged spec
 530                String lexerGrammarStr = rootGrammar.getLexerGrammar();
 531                //System.out.println("lexer rootGrammar:\n"+lexerGrammarStr);
 532                if (rootGrammar.type == Grammar.COMBINED && lexerGrammarStr != null) {
 533                    lexerGrammarFileName = rootGrammar.getImplicitlyGeneratedLexerFileName();
 534                    try {
 535                        Writer w = getOutputFile(rootGrammar, lexerGrammarFileName);
 536                        w.write(lexerGrammarStr);
 537                        w.close();
 538                    }
 539                    catch (IOException e) {
 540                        // emit different error message when creating the implicit lexer fails
 541                        // due to write permission error
 542                        exceptionWhenWritingLexerFile = true;
 543                        throw e;
 544                    }
 545                    try {
 546                        StringReader sr = new StringReader(lexerGrammarStr);
 547                        Grammar lexerGrammar = new Grammar(this);
 548                        lexerGrammar.composite.watchNFAConversion = internalOption_watchNFAConversion;
 549                        lexerGrammar.implicitLexer = true;
 550                        //lexerGrammar.setTool(this);
 551                        File lexerGrammarFullFile =
 552                            new File(getFileDirectory(lexerGrammarFileName), lexerGrammarFileName);
 553                        lexerGrammar.setFileName(lexerGrammarFullFile.toString());
 554
 555                        lexerGrammar.importTokenVocabulary(rootGrammar);
 556                        lexerGrammar.parseAndBuildAST(sr);
 557
 558                        sr.close();
 559
 560                        lexerGrammar.composite.assignTokenTypes();
 561						lexerGrammar.addRulesForSyntacticPredicates();
 562                        lexerGrammar.composite.defineGrammarSymbols();
 563                        lexerGrammar.composite.createNFAs();
 564
 565                        generateRecognizer(lexerGrammar);
 566                    }
 567                    finally {
 568                        // make sure we clean up
 569                        if (deleteTempLexer) {
 570                            File outputDir = getOutputDirectory(lexerGrammarFileName);
 571                            File outputFile = new File(outputDir, lexerGrammarFileName);
 572                            outputFile.delete();
 573                        }
 574                    }
 575                }
 576            }
 577            catch (IOException e) {
 578                if (exceptionWhenWritingLexerFile) {
 579                    ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, e);
 580                }
 581                else {
 582                    ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE,
 583                                       grammarFileName);
 584                }
 585            }
 586            catch (Exception e) {
 587                ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e);
 588            }
 589            /*
 590           finally {
 591           System.out.println("creates="+ Interval.creates);
 592           System.out.println("hits="+ Interval.hits);
 593           System.out.println("misses="+ Interval.misses);
 594           System.out.println("outOfRange="+ Interval.outOfRange);
 595           }
 596            */
 597        }
 598    }
 599
 600    public void sortGrammarFiles() throws IOException {
 601        //System.out.println("Grammar names "+getGrammarFileNames());
 602        Graph<String> g = new Graph<String>();
 603        List<String> missingFiles = new ArrayList<String>();
 604        for (String gfile : grammarFileNames) {
 605            try {
 606                GrammarSpelunker grammar = new GrammarSpelunker(inputDirectory, gfile);
 607                grammar.parse();
 608                String vocabName = grammar.getTokenVocab();
 609                String grammarName = grammar.getGrammarName();
 610                // Make all grammars depend on any tokenVocab options
 611                if ( vocabName!=null ) g.addEdge(gfile, vocabName+CodeGenerator.VOCAB_FILE_EXTENSION);
 612                // Make all generated tokens files depend on their grammars
 613                g.addEdge(grammarName+CodeGenerator.VOCAB_FILE_EXTENSION, gfile);
 614            }
 615            catch (FileNotFoundException fnfe) {
 616                ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, gfile);
 617                missingFiles.add(gfile);
 618            }
 619        }
 620        List<String> sorted = g.sort();
 621        //System.out.println("sorted="+sorted);
 622        grammarFileNames.clear(); // wipe so we can give new ordered list
 623        for (int i = 0; i < sorted.size(); i++) {
 624            String f = sorted.get(i);
 625            if ( missingFiles.contains(f) ) continue;
 626            if ( !(f.endsWith(".g") || f.endsWith(".g3")) ) continue;
 627            grammarFileNames.add(f);
 628        }
 629        //System.out.println("new grammars="+grammarFileNames);
 630    }
 631
 632    /** Get a grammar mentioned on the command-line and any delegates */
 633    public Grammar getRootGrammar(String grammarFileName)
 634        throws IOException
 635    {
 636        //ST.setLintMode(true);
 637        // grammars mentioned on command line are either roots or single grammars.
 638        // create the necessary composite in case it's got delegates; even
 639        // single grammar needs it to get token types.
 640        CompositeGrammar composite = new CompositeGrammar();
 641        Grammar grammar = new Grammar(this, grammarFileName, composite);
 642        composite.setDelegationRoot(grammar);
 643        FileReader fr;
 644        File f;
 645
 646        if (haveInputDir) {
 647            f = new File(inputDirectory, grammarFileName);
 648        }
 649        else {
 650            f = new File(grammarFileName);
 651        }
 652
 653        // Store the location of this grammar as if we import files, we can then
 654        // search for imports in the same location as the original grammar as well as in
 655        // the lib directory.
 656        //
 657        parentGrammarDirectory = f.getParent();
 658
 659        if (grammarFileName.lastIndexOf(File.separatorChar) == -1) {
 660            grammarOutputDirectory = ".";
 661        }
 662        else {
 663            grammarOutputDirectory = grammarFileName.substring(0, grammarFileName.lastIndexOf(File.separatorChar));
 664        }
 665        fr = new FileReader(f);
 666        BufferedReader br = new BufferedReader(fr);
 667        grammar.parseAndBuildAST(br);
 668        composite.watchNFAConversion = internalOption_watchNFAConversion;
 669        br.close();
 670        fr.close();
 671        return grammar;
 672    }
 673
 674    /** Create NFA, DFA and generate code for grammar.
 675     *  Create NFA for any delegates first.  Once all NFA are created,
 676     *  it's ok to create DFA, which must check for left-recursion.  That check
 677     *  is done by walking the full NFA, which therefore must be complete.
 678     *  After all NFA, comes DFA conversion for root grammar then code gen for
 679     *  root grammar.  DFA and code gen for delegates comes next.
 680     */
 681    protected void generateRecognizer(Grammar grammar) {
 682        String language = (String) grammar.getOption("language");
 683        if (language != null) {
 684            CodeGenerator generator = new CodeGenerator(this, grammar, language);
 685            grammar.setCodeGenerator(generator);
 686            generator.setDebug(isDebug());
 687            generator.setProfile(isProfile());
 688            generator.setTrace(isTrace());
 689
 690            // generate NFA early in case of crash later (for debugging)
 691            if (isGenerate_NFA_dot()) {
 692                generateNFAs(grammar);
 693            }
 694
 695            // GENERATE CODE
 696            generator.genRecognizer();
 697
 698            if (isGenerate_DFA_dot()) {
 699                generateDFAs(grammar);
 700            }
 701
 702            List<Grammar> delegates = grammar.getDirectDelegates();
 703            for (int i = 0; delegates != null && i < delegates.size(); i++) {
 704                Grammar delegate = delegates.get(i);
 705                if (delegate != grammar) { // already processing this one
 706                    generateRecognizer(delegate);
 707                }
 708            }
 709        }
 710    }
 711
 712    public void generateDFAs(Grammar g) {
 713        for (int d = 1; d <= g.getNumberOfDecisions(); d++) {
 714            DFA dfa = g.getLookaheadDFA(d);
 715            if (dfa == null) {
 716                continue; // not there for some reason, ignore
 717            }
 718            DOTGenerator dotGenerator = new DOTGenerator(g);
 719            String dot = dotGenerator.getDOT(dfa.startState);
 720            String dotFileName = g.name + "." + "dec-" + d;
 721            if (g.implicitLexer) {
 722                dotFileName = g.name + Grammar.grammarTypeToFileNameSuffix[g.type] + "." + "dec-" + d;
 723            }
 724            try {
 725                writeDOTFile(g, dotFileName, dot);
 726            } catch (IOException ioe) {
 727                ErrorManager.error(ErrorManager.MSG_CANNOT_GEN_DOT_FILE,
 728                                   dotFileName,
 729                                   ioe);
 730            }
 731        }
 732    }
 733
 734    protected void generateNFAs(Grammar g) {
 735        DOTGenerator dotGenerator = new DOTGenerator(g);
 736        Collection<Rule> rules = new HashSet<Rule>(g.getAllImportedRules());
 737        rules.addAll(g.getRules());
 738
 739        for (Rule r : rules) {
 740            try {
 741                String dot = dotGenerator.getDOT(r.startState);
 742                if (dot != null) {
 743                    writeDOTFile(g, r, dot);
 744                }
 745            } catch (IOException ioe) {
 746                ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, ioe);
 747            }
 748        }
 749    }
 750
 751    protected void writeDOTFile(Grammar g, Rule r, String dot) throws IOException {
 752        writeDOTFile(g, r.grammar.name + "." + r.name, dot);
 753    }
 754
 755    protected void writeDOTFile(Grammar g, String name, String dot) throws IOException {
 756        Writer fw = getOutputFile(g, name + ".dot");
 757        fw.write(dot);
 758        fw.close();
 759    }
 760
 761    private static void version() {
 762        ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
 763    }
 764
 765    private static void help() {
 766        ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
 767        System.err.println("usage: java org.antlr.Tool [args] file.g [file2.g file3.g ...]");
 768        System.err.println("  -o outputDir          specify output directory where all output is generated");
 769        System.err.println("  -fo outputDir         same as -o but force even files with relative paths to dir");
 770        System.err.println("  -lib dir              specify location of token files");
 771        System.err.println("  -depend               generate file dependencies");
 772        System.err.println("  -report               print out a report about the grammar(s) processed");
 773        System.err.println("  -print                print out the grammar without actions");
 774        System.err.println("  -debug                generate a parser that emits debugging events");
 775		System.err.println("  -profile              generate a parser that computes profiling information");
 776		System.err.println("  -trace                generate a recognizer that traces rule entry/exit");
 777        System.err.println("  -nfa                  generate an NFA for each rule");
 778        System.err.println("  -dfa                  generate a DFA for each decision point");
 779        System.err.println("  -message-format name  specify output style for messages");
 780        System.err.println("  -verbose              generate ANTLR version and other information");
 781        System.err.println("  -make                 only build if generated files older than grammar");
 782		System.err.println("  -version              print the version of ANTLR and exit.");
 783		System.err.println("  -language L           override language grammar option; generate L");
 784        System.err.println("  -X                    display extended argument list");
 785    }
 786
 787    private static void Xhelp() {
 788        ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
 789        System.err.println("  -Xgrtree                print the grammar AST");
 790        System.err.println("  -Xdfa                   print DFA as text ");
 791        System.err.println("  -Xnoprune               test lookahead against EBNF block exit branches");
 792        System.err.println("  -Xnocollapse            collapse incident edges into DFA states");
 793		System.err.println("  -Xdbgconversion         dump lots of info during NFA conversion");
 794		System.err.println("  -Xconversiontimeout     use to restrict NFA conversion exponentiality");
 795        System.err.println("  -Xmultithreaded         run the analysis in 2 threads");
 796        System.err.println("  -Xnomergestopstates     do not merge stop states");
 797        System.err.println("  -Xdfaverbose            generate DFA states in DOT with NFA configs");
 798        System.err.println("  -Xwatchconversion       print a message for each NFA before converting");
 799        System.err.println("  -XdbgST                 put tags at start/stop of all templates in output");
 800        System.err.println("  -Xnfastates             for nondeterminisms, list NFA states for each path");
 801        System.err.println("  -Xm m                   max number of rule invocations during conversion           [" + NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK + "]");
 802        System.err.println("  -Xmaxdfaedges m         max \"comfortable\" number of edges for single DFA state     [" + DFA.MAX_STATE_TRANSITIONS_FOR_TABLE + "]");
 803        System.err.println("  -Xmaxinlinedfastates m  max DFA states before table used rather than inlining      [" + CodeGenerator.MADSI_DEFAULT +"]");
 804        System.err.println("  -Xmaxswitchcaselabels m don't generate switch() statements for dfas bigger  than m [" + CodeGenerator.MSCL_DEFAULT +"]");
 805		System.err.println("  -Xminswitchalts m       don't generate switch() statements for dfas smaller than m [" + CodeGenerator.MSA_DEFAULT + "]");
 806		System.err.println("  -Xsavelexer             don't delete temporary lexers generated from combined grammars");
 807    }
 808
 809    /**
 810     * Set the threshold of case labels beyond which ANTLR will not instruct the target template
 811     * to generate switch() { case xxx: ...
 812     *
 813     * @param maxSwitchCaseLabels Maximum number of case lables that ANTLR should allow the target code
 814     */
 815    public void setMaxSwitchCaseLabels(int maxSwitchCaseLabels) {
 816        CodeGenerator.MAX_SWITCH_CASE_LABELS = maxSwitchCaseLabels;
 817    }
 818
 819    /**
 820     * Set the threshold of the number alts, below which ANTLR will not instruct the target
 821     * template to use a switch statement.
 822     *
 823     * @param minSwitchAlts the minimum number of alts required to use a switch staement
 824     */
 825    public void setMinSwitchAlts(int minSwitchAlts) {
 826        CodeGenerator.MIN_SWITCH_ALTS = minSwitchAlts;
 827    }
 828
 829    /**
 830     * Set the location (base directory) where output files should be produced
 831     * by the ANTLR tool.
 832     * @param outputDirectory
 833     */
 834    public void setOutputDirectory(String outputDirectory) {
 835        haveOutputDir = true;
 836        this.outputDirectory = outputDirectory;
 837    }
 838
 839    /**
 840     * Used by build tools to force the output files to always be
 841     * relative to the base output directory, even though the tool
 842     * had to set the output directory to an absolute path as it
 843     * cannot rely on the workign directory like command line invocation
 844     * can.
 845     *
 846     * @param forceRelativeOutput true if output files hould always be relative to base output directory
 847     */
 848    public void setForceRelativeOutput(boolean forceRelativeOutput) {
 849        this.forceRelativeOutput = forceRelativeOutput;
 850    }
 851
 852    /**
 853     * Set the base location of input files. Normally (when the tool is
 854     * invoked from the command line), the inputDirectory is not set, but
 855     * for build tools such as Maven, we need to be able to locate the input
 856     * files relative to the base, as the working directory could be anywhere and
 857     * changing workig directories is not a valid concept for JVMs because of threading and
 858     * so on. Setting the directory just means that the getFileDirectory() method will
 859     * try to open files relative to this input directory.
 860     *
 861     * @param inputDirectory Input source base directory
 862     */
 863    public void setInputDirectory(String inputDirectory) {
 864        this.inputDirectory = inputDirectory;
 865        haveInputDir = true;
 866    }
 867
 868    /** This method is used by all code generators to create new output
 869     *  files. If the outputDir set by -o is not present it will be created.
 870     *  The final filename is sensitive to the output directory and
 871     *  the directory where the grammar file was found.  If -o is /tmp
 872     *  and the original grammar file was foo/t.g then output files
 873     *  go in /tmp/foo.
 874     *
 875     *  The output dir -o spec takes precedence if it's absolute.
 876     *  E.g., if the grammar file dir is absolute the output dir is given
 877     *  precendence. "-o /tmp /usr/lib/t.g" results in "/tmp/T.java" as
 878     *  output (assuming t.g holds T.java).
 879     *
 880     *  If no -o is specified, then just write to the directory where the
 881     *  grammar file was found.
 882     *
 883     *  If outputDirectory==null then write a String.
 884     */
 885    public Writer getOutputFile(Grammar g, String fileName) throws IOException {
 886        if (getOutputDirectory() == null) {
 887            return new StringWriter();
 888        }
 889        // output directory is a function of where the grammar file lives
 890        // for subdir/T.g, you get subdir here.  Well, depends on -o etc...
 891        // But, if this is a .tokens file, then we force the output to
 892        // be the base output directory (or current directory if there is not a -o)
 893        //
 894        File outputDir;
 895        if (fileName.endsWith(CodeGenerator.VOCAB_FILE_EXTENSION)) {
 896            if (haveOutputDir) {
 897                outputDir = new File(getOutputDirectory());
 898            }
 899            else {
 900                outputDir = new File(".");
 901            }
 902        }
 903        else {
 904            outputDir = getOutputDirectory(g.getFileName());
 905        }
 906        File outputFile = new File(outputDir, fileName);
 907
 908        if (!outputDir.exists()) {
 909            outputDir.mkdirs();
 910        }
 911        FileWriter fw = new FileWriter(outputFile);
 912        return new BufferedWriter(fw);
 913    }
 914
 915    /**
 916     * Return the location where ANTLR will generate output files for a given file. This is a
 917     * base directory and output files will be relative to here in some cases
 918     * such as when -o option is used and input files are given relative
 919     * to the input directory.
 920     *
 921     * @param fileNameWithPath path to input source
 922     */
 923    public File getOutputDirectory(String fileNameWithPath) {
 924
 925        File outputDir;
 926        String fileDirectory;
 927
 928        // Some files are given to us without a PATH but should should
 929        // still be written to the output directory in the relative path of
 930        // the output directory. The file directory is either the set of sub directories
 931        // or just or the relative path recorded for the parent grammar. This means
 932        // that when we write the tokens files, or the .java files for imported grammars
 933        // taht we will write them in the correct place.
 934        //
 935        if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) {
 936
 937            // No path is included in the file name, so make the file
 938            // directory the same as the parent grammar (which might sitll be just ""
 939            // but when it is not, we will write the file in the correct place.
 940            //
 941            fileDirectory = grammarOutputDirectory;
 942
 943        }
 944        else {
 945            fileDirectory = fileNameWithPath.substring(0, fileNameWithPath.lastIndexOf(File.separatorChar));
 946        }
 947        if (haveOutputDir) {
 948            // -o /tmp /var/lib/t.g => /tmp/T.java
 949            // -o subdir/output /usr/lib/t.g => subdir/output/T.java
 950            // -o . /usr/lib/t.g => ./T.java
 951            if ((fileDirectory != null && !forceRelativeOutput) &&
 952                (new File(fileDirectory).isAbsolute() ||
 953                 fileDirectory.startsWith("~")) || // isAbsolute doesn't count this :(
 954                isForceAllFilesToOutputDir()) {
 955                // somebody set the dir, it takes precendence; write new file there
 956                outputDir = new File(getOutputDirectory());
 957            }
 958            else {
 959                // -o /tmp subdir/t.g => /tmp/subdir/t.g
 960                if (fileDirectory != null) {
 961                    outputDir = new File(getOutputDirectory(), fileDirectory);
 962                }
 963                else {
 964                    outputDir = new File(getOutputDirectory());
 965                }
 966            }
 967        }
 968        else {
 969            // they didn't specify a -o dir so just write to location
 970            // where grammar is, absolute or relative, this will only happen
 971            // with command line invocation as build tools will always
 972            // supply an output directory.
 973            //
 974            outputDir = new File(fileDirectory);
 975        }
 976        return outputDir;
 977    }
 978
 979    /**
 980     * Name a file from the -lib dir.  Imported grammars and .tokens files
 981     *
 982     * If we do not locate the file in the library directory, then we try
 983     * the location of the originating grammar.
 984     *
 985     * @param fileName input name we are looking for
 986     * @return Path to file that we think shuold be the import file
 987     *
 988     * @throws java.io.IOException
 989     */
 990    public String getLibraryFile(String fileName) throws IOException {
 991
 992        // First, see if we can find the file in the library directory
 993        //
 994        File f = new File(getLibraryDirectory() + File.separator + fileName);
 995
 996        if (f.exists()) {
 997
 998            // Found in the library directory
 999            //
1000            return f.getAbsolutePath();
1001        }
1002
1003        // Need to assume it is in the same location as the input file. Note that
1004        // this is only relevant for external build tools and when the input grammar
1005        // was specified relative to the source directory (working directory if using
1006        // the command line.
1007        //
1008        return parentGrammarDirectory + File.separator + fileName;
1009    }
1010
1011    /** Return the directory containing the grammar file for this grammar.
1012     *  normally this is a relative path from current directory.  People will
1013     *  often do "java org.antlr.Tool grammars/*.g3"  So the file will be
1014     *  "grammars/foo.g3" etc...  This method returns "grammars".
1015     *
1016     *  If we have been given a specific input directory as a base, then
1017     *  we must find the directory relative to this directory, unless the
1018     *  file name is given to us in absolute terms.
1019     */
1020    public String getFileDirectory(String fileName) {
1021
1022        File f;
1023        if (haveInputDir && !fileName.startsWith(File.separator)) {
1024            f = new File(inputDirectory, fileName);
1025        }
1026        else {
1027            f = new File(fileName);
1028        }
1029        // And ask Java what the base directory of this location is
1030        //
1031        return f.getParent();
1032    }
1033
1034    /** Return a File descriptor for vocab file.  Look in library or
1035     *  in -o output path.  antlr -o foo T.g U.g where U needs T.tokens
1036     *  won't work unless we look in foo too. If we do not find the
1037     *  file in the lib directory then must assume that the .tokens file
1038     *  is going to be generated as part of this build and we have defined
1039     *  .tokens files so that they ALWAYS are generated in the base output
1040     *  directory, which means the current directory for the command line tool if there
1041     *  was no output directory specified.
1042     */
1043    public File getImportedVocabFile(String vocabName) {
1044
1045        File f = new File(getLibraryDirectory(),
1046                          File.separator +
1047                          vocabName +
1048                          CodeGenerator.VOCAB_FILE_EXTENSION);
1049        if (f.exists()) {
1050            return f;
1051        }
1052
1053        // We did not find the vocab file in the lib directory, so we need
1054        // to look for it in the output directory which is where .tokens
1055        // files are generated (in the base, not relative to the input
1056        // location.)
1057        //
1058        if (haveOutputDir) {
1059            f = new File(getOutputDirectory(), vocabName + CodeGenerator.VOCAB_FILE_EXTENSION);
1060        }
1061        else {
1062            f = new File(vocabName + CodeGenerator.VOCAB_FILE_EXTENSION);
1063        }
1064        return f;
1065    }
1066
1067    /** If the tool needs to panic/exit, how do we do that?
1068     */
1069    public void panic() {
1070        throw new Error("ANTLR panic");
1071    }
1072
1073    /** Return a time stamp string accurate to sec: yyyy-mm-dd hh:mm:ss
1074     */
1075    public static String getCurrentTimeStamp() {
1076        GregorianCalendar calendar = new java.util.GregorianCalendar();
1077        int y = calendar.get(Calendar.YEAR);
1078        int m = calendar.get(Calendar.MONTH) + 1; // zero-based for months
1079        int d = calendar.get(Calendar.DAY_OF_MONTH);
1080        int h = calendar.get(Calendar.HOUR_OF_DAY);
1081        int min = calendar.get(Calendar.MINUTE);
1082        int sec = calendar.get(Calendar.SECOND);
1083        String sy = String.valueOf(y);
1084        String sm = m < 10 ? "0" + m : String.valueOf(m);
1085        String sd = d < 10 ? "0" + d : String.valueOf(d);
1086        String sh = h < 10 ? "0" + h : String.valueOf(h);
1087        String smin = min < 10 ? "0" + min : String.valueOf(min);
1088        String ssec = sec < 10 ? "0" + sec : String.valueOf(sec);
1089        return new StringBuffer().append(sy).append("-").append(sm).append("-").append(sd).append(" ").append(sh).append(":").append(smin).append(":").append(ssec).toString();
1090    }
1091
1092    /**
1093     * Provide the List of all grammar file names that the ANTLR tool will
1094     * process or has processed.
1095     *
1096     * @return the grammarFileNames
1097     */
1098    public List<String> getGrammarFileNames() {
1099        return grammarFileNames;
1100    }
1101
1102    /**
1103     * Indicates whether ANTLR has gnerated or will generate a description of
1104     * all the NFAs in <a href="http://www.graphviz.org">Dot format</a>
1105     *
1106     * @return the generate_NFA_dot
1107     */
1108    public boolean isGenerate_NFA_dot() {
1109        return generate_NFA_dot;
1110    }
1111
1112    /**
1113     * Indicates whether ANTLR has generated or will generate a description of
1114     * all the NFAs in <a href="http://www.graphviz.org">Dot format</a>
1115     *
1116     * @return the generate_DFA_dot
1117     */
1118    public boolean isGenerate_DFA_dot() {
1119        return generate_DFA_dot;
1120    }
1121
1122    /**
1123     * Return the Path to the base output directory, where ANTLR
1124     * will generate all the output files for the current language target as
1125     * well as any ancillary files such as .tokens vocab files.
1126     *
1127     * @return the output Directory
1128     */
1129    public String getOutputDirectory() {
1130        return outputDirectory;
1131    }
1132
1133    /**
1134     * Return the Path to the directory in which ANTLR will search for ancillary
1135     * files such as .tokens vocab files and imported grammar files.
1136     *
1137     * @return the lib Directory
1138     */
1139    public String getLibraryDirectory() {
1140        return libDirectory;
1141    }
1142
1143    /**
1144     * Indicate if ANTLR has generated, or will generate a debug version of the
1145     * recognizer. Debug versions of a parser communicate with a debugger such
1146     * as that contained in ANTLRWorks and at start up will 'hang' waiting for
1147     * a connection on an IP port (49100 by default).
1148     *
1149     * @return the debug flag
1150     */
1151    public boolean isDebug() {
1152        return debug;
1153    }
1154
1155    /**
1156     * Indicate whether ANTLR has generated, or will generate a version of the
1157     * recognizer that prints trace messages on entry and exit of each rule.
1158     *
1159     * @return the trace flag
1160     */
1161    public boolean isTrace() {
1162        return trace;
1163    }
1164
1165    /**
1166     * Indicates whether ANTLR has generated or will generate a version of the
1167     * recognizer that gathers statistics about its execution, which it prints when
1168     * it terminates.
1169     *
1170     * @return the profile
1171     */
1172    public boolean isProfile() {
1173        return profile;
1174    }
1175
1176    /**
1177     * Indicates whether ANTLR has generated or will generate a report of various
1178     * elements of the grammar analysis, once it it has finished analyzing a grammar
1179     * file.
1180     *
1181     * @return the report flag
1182     */
1183    public boolean isReport() {
1184        return report;
1185    }
1186
1187    /**
1188     * Indicates whether ANTLR has printed, or will print, a version of the input grammar
1189     * file(s) that is stripped of any action code embedded within.
1190     *
1191     * @return the printGrammar flag
1192     */
1193    public boolean isPrintGrammar() {
1194        return printGrammar;
1195    }
1196
1197    /**
1198     * Indicates whether ANTLR has supplied, or will supply, a list of all the things
1199     * that the input grammar depends upon and all the things that will be generated
1200     * when that grammar is successfully analyzed.
1201     *
1202     * @return the depend flag
1203     */
1204    public boolean isDepend() {
1205        return depend;
1206    }
1207
1208    /**
1209     * Indicates whether ANTLR will force all files to the output directory, even
1210     * if the input files have relative paths from the input directory.
1211     *
1212     * @return the forceAllFilesToOutputDir flag
1213     */
1214    public boolean isForceAllFilesToOutputDir() {
1215        return forceAllFilesToOutputDir;
1216    }
1217
1218    /**
1219     * Indicates whether ANTLR will be verbose when analyzing grammar files, such as
1220     * displaying the names of the files it is generating and similar information.
1221     *
1222     * @return the verbose flag
1223     */
1224    public boolean isVerbose() {
1225        return verbose;
1226    }
1227
1228    /**
1229     * Provide the current setting of the conversion timeout on DFA creation.
1230     *
1231     * @return DFA creation timeout value in milliseconds
1232     */
1233    pub

Large files files are truncated, but you can click here to view the full file