PageRenderTime 153ms CodeModel.GetById 2ms app.highlight 134ms RepoModel.GetById 1ms app.codeStats 0ms

/bundles/plugins-trunk/XML/sidekick/ecmascript/parser/EcmaScript.jj

#
Unknown | 2408 lines | 2238 code | 170 blank | 0 comment | 0 complexity | 8c8fbb89355ba15c6c823822cde55b39 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/*@bgen(jjtree) Generated By:JJTree: Do not edit this line. /home/elelay/jedit/relax-ng/sidekick/ecmascript/parser/EcmaScript.jj */
   2/*@egen*//*
   3      Copyright (c) 2004-2005, The Dojo Foundation
   4      All Rights Reserved.
   5
   6      Licensed under the Academic Free License version 2.1 or above OR the
   7      modified BSD license. For more information on Dojo licensing, see:
   8
   9      http://dojotoolkit.org/community/licensing.shtml <http://dojotoolkit.org/community/licensing.shtml>
  10
  11      Code donated to the Dojo Foundation by AOL LLC under the terms of
  12      the Dojo CCLA (http://dojotoolkit.org/ccla.txt).
  13
  14*/
  15
  16/*
  17    danson, modified for jEdit, Sep 2006.  What a nicely done jjt file!  Kudos
  18    to the original author(s).
  19*/
  20
  21
  22options {
  23
  24    /*
  25     * Default value is true. If true, all methods and class variables
  26     * are specified as static in the generated parser and token manager. This allows only
  27     * one parser object to be present, but it improves the performance of the parser.
  28     */
  29    STATIC = false;
  30
  31    /*
  32     * Options for obtaining debugging information
  33     */
  34    DEBUG_PARSER = false;
  35
  36    DEBUG_TOKEN_MANAGER = false;
  37
  38    /*
  39     * Default value is false. When set to true, the generated parser
  40     * uses an input stream object that processes Java Unicode escapes before
  41     * sending characters to the token manager.
  42     */
  43    JAVA_UNICODE_ESCAPE = true;
  44
  45    /*
  46     * Default value is false. When set to true, the generated parser
  47     * uses uses an input stream object that reads Unicode files. By default, ASCII files
  48     * are assumed.
  49     */
  50    UNICODE_INPUT = true;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         
  51
  52    /*
  53     * JDK Version
  54     */
  55    JDK_VERSION = "1.5";
  56}
  57
  58PARSER_BEGIN(EcmaScript)
  59
  60package sidekick.ecmascript.parser;
  61
  62import java.io.*;
  63import java.util.*;
  64import java.util.regex.*;
  65import sidekick.util.*;
  66
  67public class EcmaScript/*@bgen(jjtree)*/implements EcmaScriptTreeConstants/*@egen*/ {/*@bgen(jjtree)*/
  68  protected JJTEcmaScriptState jjtree = new JJTEcmaScriptState();
  69
  70/*@egen*/
  71
  72    private int lineOffset = 0;
  73    private List<ParseError> parseErrors = new ArrayList<ParseError>();
  74
  75    public static void main(String args[]){
  76        EcmaScript parser;
  77        if(args.length == 0){
  78            System.out.println("EcmaScript Parser:  Reading from standard input . . .");
  79            parser = new EcmaScript(System.in);
  80        } else if(args.length == 1){
  81            System.out.println("EcmaScript Parser:  Reading from file " + args[0] + " . . .");
  82            try {
  83                parser = new EcmaScript(new FileInputStream(args[0]), "UTF-8");
  84            } catch(java.io.FileNotFoundException e){
  85                System.out.println("EcmaScript Parser:  File " + args[0] + " not found.");
  86                return;
  87            }
  88        } else {
  89            System.out.println("EcmaScript Parser:  Usage is one of:");
  90            System.out.println("         EcmaScript < inputfile");
  91            System.out.println("OR");
  92            System.out.println("         EcmaScript inputfile");
  93            return;
  94        }
  95        try {
  96            SimpleNode n = parser.Program();
  97            System.out.println("EcmaScript parser:  EcmaScript program parsed successfully.");
  98        } catch (ParseException e) {
  99            System.out.println(e.getMessage());
 100            System.out.println("EcmaScript parser:  Encountered errors during parse.");
 101        }
 102    }
 103
 104    void jjtreeOpenNodeScope(Node n){
 105        Token t = getToken(1);
 106        if(t != null){
 107            ((SimpleNode) n).setBeginToken(t);
 108            ((SimpleNode) n).setStartLocation(getStartLocation(t));
 109        }
 110    }
 111
 112    void jjtreeCloseNodeScope(Node n){
 113        Token t = getToken(0);
 114        if(t != null){
 115            ((SimpleNode) n).setEndToken(t);
 116            ((SimpleNode) n).setEndLocation(getEndLocation(t));
 117        }
 118    }
 119
 120    public Location getStartLocation(Token t) {
 121       if (t == null)
 122           return new Location(0 + lineOffset, 0);
 123       return new Location(t.beginLine + lineOffset, t.beginColumn);
 124    }
 125
 126    public Location getEndLocation(Token t) {
 127       if (t == null)
 128           return new Location(0 + lineOffset, 0);
 129       return new Location(t.endLine + lineOffset, t.endColumn + 1);
 130    }
 131
 132    public void setLineOffset(int offset) {
 133        lineOffset = offset > 0 ? offset : 0;
 134    }
 135
 136    public void setTabSize(int size) {
 137        jj_input_stream.setTabSize(size);
 138    }
 139
 140    public int getTabSize() {
 141        return jj_input_stream.getTabSize(0);
 142    }
 143
 144    private void addException(ParseException pe) {
 145        // check for a jsp tag, these can appear in several places and will cause
 146        // a parse error, but they can be ignored.  It's easier to catch them and
 147        // ignore them than rewrite the grammar to allow them in all the appropriate
 148        // places.
 149        Token t = pe.currentToken;
 150        if (t != null) {
 151            if (t.next != null && t.next.image.startsWith("<%")) {
 152                return;
 153            }
 154        }
 155        Range range = getExceptionLocation( pe );
 156        parseErrors.add(new ParseError(pe.getMessage(), range));
 157    }
 158
 159    public List<ParseError> getParseErrors() {
 160       return parseErrors;
 161    }
 162
 163    /**
 164     * @return attempts to return a Location indicating the location of a parser
 165     * exception.  If the ParseException contains a Token reference, all is well,
 166     * otherwise, this method attempts to parse the message string for the
 167     * exception.
 168     */
 169    private Range getExceptionLocation( ParseException pe ) {
 170        Token t = pe.currentToken;
 171        if ( t != null ) {
 172            return new Range( new Location( t.next.beginLine + lineOffset, t.next.beginColumn ), new Location( t.next.endLine + lineOffset, t.next.endColumn ) );
 173        }
 174
 175        // ParseException message look like: "Parse error at line 116, column 5.  Encountered: }"
 176        try {
 177            Pattern p = Pattern.compile( "(.*?)(\\d+)(.*?)(\\d+)(.*?)" );
 178            Matcher m = p.matcher( pe.getMessage() );
 179            if ( m.matches() ) {
 180                String ln = m.group( 2 );
 181                String cn = m.group( 4 );
 182                int line_number = -1;
 183                int column_number = 0;
 184                if ( ln != null )
 185                    line_number = Integer.parseInt( ln );
 186                if ( cn != null )
 187                    column_number = Integer.parseInt( cn );
 188                return line_number > -1 ? new Range( new Location( line_number + lineOffset, column_number - 1 ), new Location( line_number + lineOffset, column_number ) ) : null;
 189            }
 190            return new Range();
 191        }
 192        catch ( Exception e ) {
 193            //e.printStackTrace();
 194            return new Range();
 195        }
 196    }
 197
 198}
 199PARSER_END(EcmaScript)
 200
 201TOKEN_MGR_DECLS : {
 202    public int htmlTokenNestingLevel = 0;
 203    public boolean expectActionScript = false;
 204}
 205
 206
 207/*****************************************
 208 * LEXICAL & REGEXP GRAMMARS STARTS HERE *
 209 *****************************************/
 210/*
 211
 212
 213
 214/* Section 7 : Lexical Conventions */
 215
 216
 217
 218/*
 219
 220TOKEN:
 221{
 222  < SOURCE_CHAR: ["\u0000"-"\ufffe"] >
 223}
 224
 225
 226MORE :
 227{
 228  < INPUT_ELEMENT_DIV:
 229      <WHITE_SPACE>
 230    | <LINE_TERMINATOR>
 231    | <COMMENT>
 232    | <_TOKEN>
 233    | <DIV_PUNCTUATOR>
 234  >
 235}
 236
 237MORE:
 238{
 239  < INPUTELEMENTREGEXP:
 240      <WHITE_SPACE>
 241    | <LINE_TERMINATOR>
 242    | <COMMENT>
 243    | <_TOKEN>
 244    | <REGULAR_EXPRESSION_LITERAL>
 245  >
 246}
 247
 248*/
 249
 250/* Section 7.2 : White Space */
 251
 252<DEFAULT, IN_REGEX>
 253SPECIAL_TOKEN :
 254{
 255    <WHITE_SPACE: <TAB> | <VT> | <FF> | <SP> | <NBSP> >
 256|
 257    < #TAB: "	" | "\t" > /* TAB */
 258|
 259    < #VT: "\u000b" > /* Vertical Tab */
 260|
 261    < #FF: "" | "\f"> /* Form Feed */
 262|
 263    < #SP: " " | " " > /* Space */
 264|
 265    < #NBSP: "\u00a0" > /* No-break space */
 266|
 267    < #USP: /* Other Unicode space seperator */
 268        ["\u2000"]
 269        | ["\u2001"]
 270        | ["\u2002"]
 271        | ["\u2003"]
 272        | ["\u2004"]
 273        | ["\u2005"]
 274        | ["\u2006"]
 275        | ["\u2007"]
 276        | ["\u2008"]
 277        | ["\u2009"]
 278        | ["\u200a"]
 279        | ["\u200b"]
 280        | ["\u3000"]
 281    >
 282}
 283
 284/* Section 7.3 : Line Terminators */
 285
 286<DEFAULT, IN_REGEX>
 287SPECIAL_TOKEN :
 288{
 289    <LINE_TERMINATOR: <LF> | <CR> | <LS> | <PS> >
 290|
 291    < #LF: "\n" > /* Line Feed */
 292|
 293    < #CR: "\r" > /* Carriage Return */
 294|
 295    < #LS: "\u2028" > /* Line separator */
 296|
 297    < #PS: "\u2029" > /* Paragraph separator */
 298}
 299
 300
 301/* Comments */
 302
 303<DEFAULT, IN_REGEX>
 304MORE :
 305{
 306    "//" : IN_SINGLE_LINE_COMMENT
 307|
 308    "/*" : IN_MULTI_LINE_COMMENT
 309|
 310    "<%" : IN_JSP_TAG       // not necessarily a comment...
 311}
 312
 313<IN_SINGLE_LINE_COMMENT>
 314SPECIAL_TOKEN :
 315{
 316    <SINGLE_LINE_COMMENT: (~["\n","\r"])* ("\n"|"\r"|"\r\n")? > : DEFAULT
 317}
 318
 319<IN_MULTI_LINE_COMMENT>
 320SPECIAL_TOKEN :
 321{
 322    <MULTI_LINE_COMMENT: "*/" > : DEFAULT
 323}
 324
 325<IN_JSP_TAG>
 326SPECIAL_TOKEN :
 327{
 328    <JSP_TAG: "%>" > : DEFAULT
 329}
 330
 331<IN_SINGLE_LINE_COMMENT, IN_MULTI_LINE_COMMENT, IN_JSP_TAG, IN_PATTERN>
 332MORE :
 333{
 334    < ~[] >
 335}
 336
 337
 338
 339/* Section 7.5.1: Reserved Words */
 340
 341<DEFAULT, IN_REGEX>
 342TOKEN :
 343{
 344    < BREAK: "break" > : DEFAULT
 345|
 346    < CONTINUE: "continue" > : DEFAULT
 347|
 348    < DELETE: "delete" > : DEFAULT
 349|
 350    < ELSE: "else" > : DEFAULT
 351|
 352    < FOR: "for" > : DEFAULT
 353|
 354    < FUNCTION: "function" > : DEFAULT
 355|
 356    < IF: "if" > : DEFAULT
 357|
 358    < IN: "in" > : DEFAULT
 359|
 360    < NEW: "new" > : DEFAULT
 361|
 362    < RETURN: "return" > : DEFAULT
 363|
 364    < THIS: "this" > : IN_REGEX
 365|
 366    < TYPEOF: "typeof" > : DEFAULT
 367|
 368    < VAR: "var" > : DEFAULT
 369|
 370    < VOID: "void" > : DEFAULT
 371|
 372    < WHILE: "while" > : DEFAULT
 373|
 374    < WITH: "with" > : DEFAULT
 375|
 376    < CASE: "case" > : DEFAULT
 377|
 378    < CATCH: "catch" > : DEFAULT
 379|
 380    < CLASS: "class" > : DEFAULT
 381|
 382    < CONST: "const" > : DEFAULT
 383|
 384    < DEBUGGER: "debugger" > : DEFAULT
 385|
 386    < _DEFAULT: "default" > : DEFAULT
 387|
 388    < DO: "do" > : DEFAULT
 389|
 390    < ENUM: "enum" > : DEFAULT
 391|
 392    < EXPORT: "export" > : DEFAULT
 393|
 394    < EXTENDS: "extends" > : DEFAULT
 395|
 396    < FINALLY: "finally" > : DEFAULT
 397|
 398    < IMPORT: "import" > : DEFAULT
 399|
 400    < SUPER: "super" > : DEFAULT
 401|
 402    < SWITCH: "switch" > : DEFAULT
 403|
 404    < THROW: "throw" > : DEFAULT
 405|
 406    < TRY: "try" > : DEFAULT
 407}
 408
 409
 410/* JScript .NET Tokens
 411
 412TOKEN :
 413{
 414    < BYTE: "byte" >
 415    | < SBYTE: "sbyte" >
 416    | < SHORT: "short" >
 417    | < USHORT: "ushort" >
 418    | < UINT: "uint" >
 419    | < LONG: "long" >
 420    | < ULONG: "ulong" >
 421    | < FLOAT: "float" >
 422    | < NUMBER: "Number" >
 423    | < DOUBLE: "double" >
 424    | < DECIMAL: "decimal" >
 425    | < BOOLEAN: "boolean" >
 426    | < STRING: "String" >
 427    | < CHAR: "char" >
 428}
 429
 430*/
 431
 432
 433/* Section 7.7: Punctuators */
 434
 435<DEFAULT, IN_REGEX>
 436TOKEN :
 437{
 438    < LBRACE: "{" > : DEFAULT
 439|
 440    < RBRACE: "}" > : IN_REGEX
 441|
 442    < LPAREN: "(" > : DEFAULT
 443|
 444    < RPAREN: ")" > : IN_REGEX
 445|
 446    < LBRACKET: "[" > : DEFAULT
 447|
 448    < RBRACKET: "]" > : IN_REGEX
 449|
 450    < DOT: "." > : DEFAULT
 451|
 452    < SEMICOLON: ";" > : DEFAULT
 453|
 454    < COMMA: "," > : DEFAULT
 455|
 456    < LT: "<" > : DEFAULT
 457|
 458    < GT: ">" > : DEFAULT
 459|
 460    < LE: "<=" > : DEFAULT
 461|
 462    < GE: ">=" > : DEFAULT
 463|
 464    < EQ: "==" > : DEFAULT
 465|
 466    < NE: "!=" > : DEFAULT
 467|
 468    < SEQ: "===" > : DEFAULT /* Strict Equals Operator */
 469|
 470    < SNEQ: "!==" >	 : DEFAULT /* Strict Does-not-equal Operator */
 471|
 472    < PLUS: "+" > : DEFAULT
 473|
 474    < MINUS: "-" > : DEFAULT
 475|
 476    < STAR: "*" > : DEFAULT
 477|
 478    < REM: "%" > : DEFAULT
 479|
 480    < INCR: "++" > : IN_REGEX
 481|
 482    < DECR: "--" > : IN_REGEX
 483|
 484    < LSHIFT: "<<" > : DEFAULT
 485|
 486    < RSHIFT: ">>" > : DEFAULT
 487|
 488    < RUNSHIFT: ">>>" > : DEFAULT /* Unsigned Right Shift Operator */
 489|
 490    < BIT_AND: "&" > : DEFAULT
 491|
 492    < BIT_OR: "|" > : DEFAULT
 493|
 494    < XOR: "^" > : DEFAULT
 495|
 496    < BANG: "!" > : DEFAULT
 497|
 498    < TILDE: "~" > : IN_REGEX
 499|
 500    < SC_AND: "&&" > : DEFAULT
 501|
 502    < SC_OR: "||" > : DEFAULT
 503|
 504    < HOOK: "?" > : DEFAULT
 505|
 506    < COLON: ":" > : DEFAULT
 507|
 508    < ASSIGN: "=" > : DEFAULT
 509|
 510    < PLUSASSIGN: "+=" > : DEFAULT
 511|
 512    < MINUSASSIGN: "-=" > : DEFAULT
 513|
 514    < STARASSIGN: "*=" > : DEFAULT
 515|
 516    < REMASSIGN: "%=" > : DEFAULT
 517|
 518    < LSHIFTASSIGN: "<<=" > : DEFAULT
 519|
 520    < RSIGNEDSHIFTASSIGN: ">>=" > : DEFAULT
 521|
 522    < RUNSIGNEDSHIFTASSIGN: ">>>=" > : DEFAULT
 523|
 524    < ANDASSIGN: "&=" > : DEFAULT
 525|
 526    < ORASSIGN: "|=" > : DEFAULT
 527|
 528    < XORASSIGN: "^=" > : DEFAULT
 529|
 530    < INTANCE_OF: "instanceof" > : DEFAULT
 531}
 532
 533
 534/* Section 7.8.3: Numeric Literals */
 535
 536<DEFAULT, IN_REGEX>
 537TOKEN:
 538{
 539    < DECIMAL_LITERAL :
 540        <DECIMAL_INTEGER_LITERAL> "." (<DECIMAL_DIGITS>)? (<EXPONENT_PART>)?
 541    |
 542        "." <DECIMAL_DIGITS> (<EXPONENT_PART>)?
 543    |
 544        <DECIMAL_INTEGER_LITERAL> (<EXPONENT_PART>)?
 545    >  : IN_REGEX
 546|
 547    < #NON_ZERO_DIGIT: ["1"-"9"] >
 548|
 549    < #EXPONENT_PART: ("e" | "E") (["+","-"])? <DECIMAL_DIGITS> >
 550}
 551
 552<DEFAULT, IN_REGEX>
 553TOKEN:
 554{
 555    < DECIMAL_INTEGER_LITERAL:
 556        "0" | <NON_ZERO_DIGIT> (<DECIMAL_DIGITS>)?
 557    > : IN_REGEX
 558}
 559
 560<DEFAULT, IN_REGEX>
 561TOKEN:
 562{
 563    < HEX_INTEGER_LITERAL: "0" ["x","X"] (<HEX_DIGIT>)+ > : IN_REGEX
 564}
 565
 566<DEFAULT, IN_REGEX>
 567TOKEN:
 568{  < DECIMAL_DIGITS: (<DECIMAL_DIGIT>)+ > : IN_REGEX }
 569
 570TOKEN:
 571{
 572    < DECIMAL_DIGIT: ["0"-"9"] >
 573}
 574
 575
 576/* Section 7.8: Literals */
 577/*
 578
 579MORE:
 580{
 581  < LITERAL:
 582      <NULL_LITERAL>
 583    | <BOOLEAN_LITERAL>
 584    | <NUMERIC_LITERAL>
 585    | <STRING_LITERAL>
 586  >
 587}
 588
 589*/
 590
 591
 592/* Section 7.8.1: NULL Literals */
 593
 594<DEFAULT, IN_REGEX>
 595TOKEN:
 596{
 597    < NULL_LITERAL: "null" > : IN_REGEX
 598}
 599
 600/* Section 7.8.2: Boolean Literals */
 601
 602<DEFAULT, IN_REGEX>
 603TOKEN:
 604{
 605    < BOOLEAN_LITERAL: "true" | "false" > : IN_REGEX
 606}
 607
 608
 609
 610/* Section 7.8.4: String Literals */
 611
 612<DEFAULT, IN_REGEX>
 613TOKEN:
 614{
 615    < STRING_LITERAL:
 616        "\"" (<DOUBLE_STRING_CHARACTERS>)? "\""  | "'" (<SINGLE_STRING_CHARACTERS>)? "'"
 617    >	 : IN_REGEX
 618|
 619    < #DOUBLE_STRING_CHARACTERS: (<DOUBLE_STRING_CHARACTER>)* >
 620|
 621    < #SINGLE_STRING_CHARACTERS: (<SINGLE_STRING_CHARACTER>)* >
 622|
 623    < #DOUBLE_STRING_CHARACTER:
 624        (~["\"","\\","\n","\r","\u2028","\u2029"])*
 625        | "\\" <ESCAPE_SEQUENCE>
 626    >
 627|
 628    < #SINGLE_STRING_CHARACTER:
 629        (~["'","\\","\n","\r","\u2028","\u2029"])
 630        | "\\" <ESCAPE_SEQUENCE>
 631    >
 632|
 633    < #ESCAPE_SEQUENCE:
 634        <CHARACTER_ESCAPE_SEQUENCE>
 635        |
 636            "0"
 637        |
 638            <HEX_ESCAPE_SEQUENCE>
 639        |
 640            <UNICODE_ESCAPE_SEQUENCE>
 641    >
 642|
 643    < #CHARACTER_ESCAPE_SEQUENCE:
 644        <SINGLE_ESCAPE_CHARACTER> | <NON_ESCAPE_CHARACTER>
 645    >
 646|
 647    < #SINGLE_ESCAPE_CHARACTER: ["'" , "\"" , "\\" , "b" , "f" , "n" , "r" , "t" , "v"] >
 648|
 649    < #NON_ESCAPE_CHARACTER:
 650        ~["\n","\r","\u2028","\u2029"]
 651    |
 652        ~["'" , "\"" , "\\" , "b" , "f" , "n" , "r" , "t" , "v", "x", "u"]
 653    |
 654        ~["0"-"9"]
 655    >
 656}
 657
 658TOKEN:
 659{
 660    < HEX_ESCAPE_SEQUENCE: "x" <HEX_DIGIT> <HEX_DIGIT> >
 661}
 662
 663/*
 664TOKEN:
 665{
 666  < ESCAPE_CHARACTER:
 667      ["'" , "\"" , "\\" , "b" , "f" , "n" , "r" , "t" , "v"]
 668    | ["0"-"9"]
 669    | "x"
 670    | "u"
 671  >
 672}
 673*/
 674
 675/* Section 7.6: Identifiers */
 676
 677<DEFAULT, IN_REGEX>
 678TOKEN:
 679{
 680    < IDENTIFIER_NAME: <IDENTIFIER_START> (<IDENTIFIER_PART>)* > : IN_REGEX
 681|
 682    < #IDENTIFIER_START:
 683        <UNICODE_LETTER>
 684    |
 685        <DOLLAR_SIGN>
 686    |
 687        <UNDER_SCORE>
 688    |
 689        <UNICODE_ESCAPE_SEQUENCE>
 690    >
 691|
 692    < #IDENTIFIER_PART:
 693        <IDENTIFIER_START>
 694    |
 695        <UNICODE_COMBINING_MARK>
 696    |
 697        <UNICODE_DIGIT>
 698    |
 699        <UNICODE_CONNECTOR_PUNCTUATION>
 700    |
 701        <UNICODE_ESCAPE_SEQUENCE>
 702    >
 703|
 704    < #DOLLAR_SIGN: "$" >
 705|
 706    < #UNDER_SCORE: "_" >
 707|
 708    < #UNICODE_LETTER:
 709        ["A"-"Z"]
 710        | ["a"-"z"]
 711        | ["A"-"Z"]
 712        | ["a"-"z"]
 713        | ["\u00aa"]
 714        | ["\u00b5"]
 715        | ["\u00ba"]
 716        | ["\u00c0"-"\u00d6"]
 717        | ["\u00d8"-"\u00f6"]
 718        | ["\u00f8"-"\u021f"]
 719        | ["\u0222"-"\u0233"]
 720        | ["\u0250"-"\u02ad"]
 721        | ["\u02b0"-"\u02b8"]
 722        | ["\u02bb"-"\u02c1"]
 723        | ["\u02d0"-"\u02d1"]
 724        | ["\u02e0"-"\u02e4"]
 725        | ["\u02ee"]
 726        | ["\u037a"]
 727        | ["\u0386"]
 728        | ["\u0388"-"\u038a"]
 729        | ["\u038c"]
 730        | ["\u038e"-"\u03a1"]
 731        | ["\u03a3"-"\u03ce"]
 732        | ["\u03d0"-"\u03d7"]
 733        | ["\u03da"-"\u03f3"]
 734        | ["\u0400"-"\u0481"]
 735        | ["\u048c"-"\u04c4"]
 736        | ["\u04c7"-"\u04c8"]
 737        | ["\u04cb"-"\u04cc"]
 738        | ["\u04d0"-"\u04f5"]
 739        | ["\u04f8"-"\u04f9"]
 740        | ["\u0531"-"\u0556"]
 741        | ["\u0559"]
 742        | ["\u0561"-"\u0587"]
 743        | ["\u05d0"-"\u05ea"]
 744        | ["\u05f0"-"\u05f2"]
 745        | ["\u0621"-"\u063a"]
 746        | ["\u0640"-"\u064a"]
 747        | ["\u0671"-"\u06d3"]
 748        | ["\u06d5"]
 749        | ["\u06e5"-"\u06e6"]
 750        | ["\u06fa"-"\u06fc"]
 751        | ["\u0710"]
 752        | ["\u0712"-"\u072c"]
 753        | ["\u0780"-"\u07a5"]
 754        | ["\u0905"-"\u0939"]
 755        | ["\u093d"]
 756        | ["\u0950"]
 757        | ["\u0958"-"\u0961"]
 758        | ["\u0985"-"\u098c"]
 759        | ["\u098f"-"\u0990"]
 760        | ["\u0993"-"\u09a8"]
 761        | ["\u09aa"-"\u09b0"]
 762        | ["\u09b2"]
 763        | ["\u09b6"-"\u09b9"]
 764        | ["\u09dc"-"\u09dd"]
 765        | ["\u09df"-"\u09e1"]
 766        | ["\u09f0"-"\u09f1"]
 767        | ["\u0a05"-"\u0a0a"]
 768        | ["\u0a0f"-"\u0a10"]
 769        | ["\u0a13"-"\u0a28"]
 770        | ["\u0a2a"-"\u0a30"]
 771        | ["\u0a32"-"\u0a33"]
 772        | ["\u0a35"-"\u0a36"]
 773        | ["\u0a38"-"\u0a39"]
 774        | ["\u0a59"-"\u0a5c"]
 775        | ["\u0a5e"]
 776        | ["\u0a72"-"\u0a74"]
 777        | ["\u0a85"-"\u0a8b"]
 778        | ["\u0a8d"]
 779        | ["\u0a8f"-"\u0a91"]
 780        | ["\u0a93"-"\u0aa8"]
 781        | ["\u0aaa"-"\u0ab0"]
 782        | ["\u0ab2"-"\u0ab3"]
 783        | ["\u0ab5"-"\u0ab9"]
 784        | ["\u0abd"]
 785        | ["\u0ad0"]
 786        | ["\u0ae0"]
 787        | ["\u0b05"-"\u0b0c"]
 788        | ["\u0b0f"-"\u0b10"]
 789        | ["\u0b13"-"\u0b28"]
 790        | ["\u0b2a"-"\u0b30"]
 791        | ["\u0b32"-"\u0b33"]
 792        | ["\u0b36"-"\u0b39"]
 793        | ["\u0b3d"]
 794        | ["\u0b5c"-"\u0b5d"]
 795        | ["\u0b5f"-"\u0b61"]
 796        | ["\u0b85"-"\u0b8a"]
 797        | ["\u0b8e"-"\u0b90"]
 798        | ["\u0b92"-"\u0b95"]
 799        | ["\u0b99"-"\u0b9a"]
 800        | ["\u0b9c"]
 801        | ["\u0b9e"-"\u0b9f"]
 802        | ["\u0ba3"-"\u0ba4"]
 803        | ["\u0ba8"-"\u0baa"]
 804        | ["\u0bae"-"\u0bb5"]
 805        | ["\u0bb7"-"\u0bb9"]
 806        | ["\u0c05"-"\u0c0c"]
 807        | ["\u0c0e"-"\u0c10"]
 808        | ["\u0c12"-"\u0c28"]
 809        | ["\u0c2a"-"\u0c33"]
 810        | ["\u0c35"-"\u0c39"]
 811        | ["\u0c60"-"\u0c61"]
 812        | ["\u0c85"-"\u0c8c"]
 813        | ["\u0c8e"-"\u0c90"]
 814        | ["\u0c92"-"\u0ca8"]
 815        | ["\u0caa"-"\u0cb3"]
 816        | ["\u0cb5"-"\u0cb9"]
 817        | ["\u0cde"]
 818        | ["\u0ce0"-"\u0ce1"]
 819        | ["\u0d05"-"\u0d0c"]
 820        | ["\u0d0e"-"\u0d10"]
 821        | ["\u0d12"-"\u0d28"]
 822        | ["\u0d2a"-"\u0d39"]
 823        | ["\u0d60"-"\u0d61"]
 824        | ["\u0d85"-"\u0d96"]
 825        | ["\u0d9a"-"\u0db1"]
 826        | ["\u0db3"-"\u0dbb"]
 827        | ["\u0dbd"]
 828        | ["\u0dc0"-"\u0dc6"]
 829        | ["\u0e01"-"\u0e30"]
 830        | ["\u0e32"-"\u0e33"]
 831        | ["\u0e40"-"\u0e46"]
 832        | ["\u0e81"-"\u0e82"]
 833        | ["\u0e84"]
 834        | ["\u0e87"-"\u0e88"]
 835        | ["\u0e8a"]
 836        | ["\u0e8d"]
 837        | ["\u0e94"-"\u0e97"]
 838        | ["\u0e99"-"\u0e9f"]
 839        | ["\u0ea1"-"\u0ea3"]
 840        | ["\u0ea5"]
 841        | ["\u0ea7"]
 842        | ["\u0eaa"-"\u0eab"]
 843        | ["\u0ead"-"\u0eb0"]
 844        | ["\u0eb2"-"\u0eb3"]
 845        | ["\u0ebd"-"\u0ec4"]
 846        | ["\u0ec6"]
 847        | ["\u0edc"-"\u0edd"]
 848        | ["\u0f00"]
 849        | ["\u0f40"-"\u0f6a"]
 850        | ["\u0f88"-"\u0f8b"]
 851        | ["\u1000"-"\u1021"]
 852        | ["\u1023"-"\u1027"]
 853        | ["\u1029"-"\u102a"]
 854        | ["\u1050"-"\u1055"]
 855        | ["\u10a0"-"\u10c5"]
 856        | ["\u10d0"-"\u10f6"]
 857        | ["\u1100"-"\u1159"]
 858        | ["\u115f"-"\u11a2"]
 859        | ["\u11a8"-"\u11f9"]
 860        | ["\u1200"-"\u1206"]
 861        | ["\u1208"-"\u1246"]
 862        | ["\u1248"]
 863        | ["\u124a"-"\u124d"]
 864        | ["\u1250"-"\u1256"]
 865        | ["\u1258"]
 866        | ["\u125a"-"\u125d"]
 867        | ["\u1260"-"\u1286"]
 868        | ["\u1288"]
 869        | ["\u128a"-"\u128d"]
 870        | ["\u1290"-"\u12ae"]
 871        | ["\u12b0"]
 872        | ["\u12b2"-"\u12b5"]
 873        | ["\u12b8"-"\u12be"]
 874        | ["\u12c0"]
 875        | ["\u12c2"-"\u12c5"]
 876        | ["\u12c8"-"\u12ce"]
 877        | ["\u12d0"-"\u12d6"]
 878        | ["\u12d8"-"\u12ee"]
 879        | ["\u12f0"-"\u130e"]
 880        | ["\u1310"]
 881        | ["\u1312"-"\u1315"]
 882        | ["\u1318"-"\u131e"]
 883        | ["\u1320"-"\u1346"]
 884        | ["\u1348"-"\u135a"]
 885        | ["\u13a0"-"\u13b0"]
 886        | ["\u13b1"-"\u13f4"]
 887        | ["\u1401"-"\u1676"]
 888        | ["\u1681"-"\u169a"]
 889        | ["\u16a0"-"\u16ea"]
 890        | ["\u1780"-"\u17b3"]
 891        | ["\u1820"-"\u1877"]
 892        | ["\u1880"-"\u18a8"]
 893        | ["\u1e00"-"\u1e9b"]
 894        | ["\u1ea0"-"\u1ee0"]
 895        | ["\u1ee1"-"\u1ef9"]
 896        | ["\u1f00"-"\u1f15"]
 897        | ["\u1f18"-"\u1f1d"]
 898        | ["\u1f20"-"\u1f39"]
 899        | ["\u1f3a"-"\u1f45"]
 900        | ["\u1f48"-"\u1f4d"]
 901        | ["\u1f50"-"\u1f57"]
 902        | ["\u1f59"]
 903        | ["\u1f5b"]
 904        | ["\u1f5d"]
 905        | ["\u1f5f"-"\u1f7d"]
 906        | ["\u1f80"-"\u1fb4"]
 907        | ["\u1fb6"-"\u1fbc"]
 908        | ["\u1fbe"]
 909        | ["\u1fc2"-"\u1fc4"]
 910        | ["\u1fc6"-"\u1fcc"]
 911        | ["\u1fd0"-"\u1fd3"]
 912        | ["\u1fd6"-"\u1fdb"]
 913        | ["\u1fe0"-"\u1fec"]
 914        | ["\u1ff2"-"\u1ff4"]
 915        | ["\u1ff6"-"\u1ffc"]
 916        | ["\u207f"]
 917        | ["\u2102"]
 918        | ["\u2107"]
 919        | ["\u210a"-"\u2113"]
 920        | ["\u2115"]
 921        | ["\u2119"-"\u211d"]
 922        | ["\u2124"]
 923        | ["\u2126"]
 924        | ["\u2128"]
 925        | ["\u212a"-"\u212d"]
 926        | ["\u212f"-"\u2131"]
 927        | ["\u2133"-"\u2139"]
 928        | ["\u2160"-"\u2183"]
 929        | ["\u3005"-"\u3007"]
 930        | ["\u3021"-"\u3029"]
 931        | ["\u3031"-"\u3035"]
 932        | ["\u3038"-"\u303a"]
 933        | ["\u3041"-"\u3094"]
 934        | ["\u309d"-"\u309e"]
 935        | ["\u30a1"-"\u30fa"]
 936        | ["\u30fc"-"\u30fe"]
 937        | ["\u3105"-"\u312c"]
 938        | ["\u3131"-"\u318e"]
 939        | ["\u31a0"-"\u31b7"]
 940        | ["\u3400"]
 941        | ["\u4db5"]
 942        | ["\u4e00"]
 943        | ["\u9fa5"]
 944        | ["\ua000"-"\ua48c"]
 945        | ["\uac00"]
 946        | ["\ud7a3"]
 947        | ["\uf900"-"\ufa2d"]
 948        | ["\ufb00"-"\ufb06"]
 949        | ["\ufb13"-"\ufb17"]
 950        | ["\ufb1d"]
 951        | ["\ufb1f"-"\ufb28"]
 952        | ["\ufb2a"-"\ufb36"]
 953        | ["\ufb38"-"\ufb3c"]
 954        | ["\ufb3e"]
 955        | ["\ufb40"-"\ufb41"]
 956        | ["\ufb43"-"\ufb44"]
 957        | ["\ufb46"-"\ufbb1"]
 958        | ["\ufbd3"-"\ufd3d"]
 959        | ["\ufd50"-"\ufd8f"]
 960        | ["\ufd92"-"\ufdc7"]
 961        | ["\ufdf0"-"\ufdfb"]
 962        | ["\ufe70"-"\ufe72"]
 963        | ["\ufe74"]
 964        | ["\ufe76"-"\ufefc"]
 965        | ["\uff21"-"\uff3a"]
 966        | ["\uff41"-"\uff5a"]
 967        | ["\uff66"-"\uffbe"]
 968        | ["\uffc2"-"\uffc7"]
 969        | ["\uffca"-"\uffcf"]
 970        | ["\uffd2"-"\uffd7"]
 971        | ["\uffda"-"\uffdc"]
 972    >
 973}
 974
 975/*
 976 * Unicode categories Non-spacing mark (MN) OR Combining spacing mark (MC)
 977 */
 978MORE:
 979{
 980    < UNICODE_COMBINING_MARK: <MN> | <MC> >
 981}
 982
 983
 984TOKEN:
 985{
 986    < MC:
 987        ["\u0903"]
 988        | ["\u093e"]
 989        | ["\u093f"]
 990        | ["\u0940"]
 991        | ["\u0949"]
 992        | ["\u094a"]
 993        | ["\u094b"]
 994        | ["\u094c"]
 995        | ["\u0982"]
 996        | ["\u0983"]
 997        | ["\u09be"]
 998        | ["\u09bf"]
 999        | ["\u09c0"]
1000        | ["\u09c7"]
1001        | ["\u09c8"]
1002        | ["\u09cb"]
1003        | ["\u09cc"]
1004        | ["\u09d7"]
1005        | ["\u0a03"]
1006        | ["\u0a3e"]
1007        | ["\u0a3f"]
1008        | ["\u0a40"]
1009        | ["\u0a83"]
1010        | ["\u0abe"]
1011        | ["\u0abf"]
1012        | ["\u0ac0"]
1013        | ["\u0ac9"]
1014        | ["\u0acb"]
1015        | ["\u0acc"]
1016        | ["\u0b02"]
1017        | ["\u0b03"]
1018        | ["\u0b3e"]
1019        | ["\u0b40"]
1020        | ["\u0b47"]
1021        | ["\u0b48"]
1022        | ["\u0b4b"]
1023        | ["\u0b4c"]
1024        | ["\u0b57"]
1025        | ["\u0bbe"]
1026        | ["\u0bbf"]
1027        | ["\u0bc1"]
1028        | ["\u0bc2"]
1029        | ["\u0bc6"]
1030        | ["\u0bc7"]
1031        | ["\u0bc8"]
1032        | ["\u0bca"]
1033        | ["\u0bcb"]
1034        | ["\u0bcc"]
1035        | ["\u0bd7"]
1036        | ["\u0c01"]
1037        | ["\u0c02"]
1038        | ["\u0c03"]
1039        | ["\u0c41"]
1040        | ["\u0c42"]
1041        | ["\u0c43"]
1042        | ["\u0c44"]
1043        | ["\u0c82"]
1044        | ["\u0c83"]
1045        | ["\u0cbe"]
1046        | ["\u0cc0"]
1047        | ["\u0cc1"]
1048        | ["\u0cc2"]
1049        | ["\u0cc3"]
1050        | ["\u0cc4"]
1051        | ["\u0cc7"]
1052        | ["\u0cc8"]
1053        | ["\u0cca"]
1054        | ["\u0ccb"]
1055        | ["\u0cd5"]
1056        | ["\u0cd6"]
1057        | ["\u0d02"]
1058        | ["\u0d03"]
1059        | ["\u0d3e"]
1060        | ["\u0d3f"]
1061        | ["\u0d40"]
1062        | ["\u0d46"]
1063        | ["\u0d47"]
1064        | ["\u0d48"]
1065        | ["\u0d4a"]
1066        | ["\u0d4b"]
1067        | ["\u0d4c"]
1068        | ["\u0d57"]
1069        | ["\u0d82"]
1070        | ["\u0d83"]
1071        | ["\u0dcf"]
1072        | ["\u0dd0"]
1073        | ["\u0dd1"]
1074        | ["\u0dd8"]
1075        | ["\u0dd9"]
1076        | ["\u0dda"]
1077        | ["\u0ddb"]
1078        | ["\u0ddc"]
1079        | ["\u0ddd"]
1080        | ["\u0dde"]
1081        | ["\u0ddf"]
1082        | ["\u0df2"]
1083        | ["\u0df3"]
1084        | ["\u0f3e"]
1085        | ["\u0f3f"]
1086        | ["\u0f7f"]
1087        | ["\u102c"]
1088        | ["\u1031"]
1089        | ["\u1038"]
1090        | ["\u1056"]
1091        | ["\u1057"]
1092        | ["\u17b6"]
1093        | ["\u17be"]
1094        | ["\u17bf"]
1095        | ["\u17c0"]
1096        | ["\u17c1"]
1097        | ["\u17c2"]
1098        | ["\u17c3"]
1099        | ["\u17c4"]
1100        | ["\u17c5"]
1101        | ["\u17c7"]
1102        | ["\u17c8"]
1103        | ["\u1923"]
1104        | ["\u1924"]
1105        | ["\u1925"]
1106        | ["\u1926"]
1107        | ["\u1929"]
1108        | ["\u192a"]
1109        | ["\u192b"]
1110        | ["\u1930"]
1111        | ["\u1931"]
1112        | ["\u1933"]
1113        | ["\u1934"]
1114        | ["\u1935"]
1115        | ["\u1936"]
1116        | ["\u1937"]
1117        | ["\u1938"]
1118        | ["\u19b0"]
1119        | ["\u19b1"]
1120        | ["\u19b2"]
1121        | ["\u19b3"]
1122        | ["\u19b4"]
1123        | ["\u19b5"]
1124        | ["\u19b6"]
1125        | ["\u19b7"]
1126        | ["\u19b8"]
1127        | ["\u19b9"]
1128        | ["\u19ba"]
1129        | ["\u19bb"]
1130        | ["\u19bc"]
1131        | ["\u19bd"]
1132        | ["\u19be"]
1133        | ["\u19bf"]
1134        | ["\u19c0"]
1135        | ["\u19c8"]
1136        | ["\u19c9"]
1137        | ["\u1a19"]
1138        | ["\u1a1a"]
1139        | ["\u1a1b"]
1140        | ["\ua802"]
1141        | ["\ua823"]
1142        | ["\ua824"]
1143        | ["\ua827"]
1144        | ["\u1d16"]
1145        | ["\u1d16"]
1146        | ["\u1d16"]
1147        | ["\u1d16"]
1148        | ["\u1d16"]
1149        | ["\u1d17"]
1150        | ["\u1d17"]
1151        | ["\u1d17"]
1152    >
1153|
1154    < MN:
1155        ["\u0300"-"\u034e"]
1156        | ["\u0360"-"\u0362"]
1157        | ["\u0483"-"\u0486"]
1158        | ["\u0591"-"\u05a1"]
1159        | ["\u05a3"-"\u05b9"]
1160        | ["\u05bb"-"\u05bd"]
1161        | ["\u05bf"]
1162        | ["\u05c1"-"\u05c2"]
1163        | ["\u05c4"]
1164        | ["\u064b"-"\u0655"]
1165        | ["\u0670"]
1166        | ["\u06d6"-"\u06dc"]
1167        | ["\u06df"-"\u06e4"]
1168        | ["\u06e7"-"\u06e8"]
1169        | ["\u06ea"-"\u06ed"]
1170        | ["\u0711"]
1171        | ["\u0730"-"\u074a"]
1172        | ["\u07a6"-"\u07b0"]
1173        | ["\u0901"-"\u0903"]
1174        | ["\u093c"]
1175        | ["\u093e"-"\u094d"]
1176        | ["\u0951"-"\u0954"]
1177        | ["\u0962"-"\u0963"]
1178        | ["\u0981"-"\u0983"]
1179        | ["\u09bc"-"\u09c4"]
1180        | ["\u09c7"-"\u09c8"]
1181        | ["\u09cb"-"\u09cd"]
1182        | ["\u09d7"]
1183        | ["\u09e2"-"\u09e3"]
1184        | ["\u0a02"]
1185        | ["\u0a3c"]
1186        | ["\u0a3e"-"\u0a42"]
1187        | ["\u0a47"-"\u0a48"]
1188        | ["\u0a4b"-"\u0a4d"]
1189        | ["\u0a70"-"\u0a71"]
1190        | ["\u0a81"-"\u0a83"]
1191        | ["\u0abc"]
1192        | ["\u0abe"-"\u0ac5"]
1193        | ["\u0ac7"-"\u0ac9"]
1194        | ["\u0acb"-"\u0acd"]
1195        | ["\u0b01"-"\u0b03"]
1196        | ["\u0b3c"]
1197        | ["\u0b3e"-"\u0b43"]
1198        | ["\u0b47"-"\u0b48"]
1199        | ["\u0b4b"-"\u0b4d"]
1200        | ["\u0b56"-"\u0b57"]
1201        | ["\u0b82"-"\u0b83"]
1202        | ["\u0bbe"-"\u0bc2"]
1203        | ["\u0bc6"-"\u0bc8"]
1204        | ["\u0bca"-"\u0bcd"]
1205        | ["\u0bd7"]
1206        | ["\u0c01"-"\u0c03"]
1207        | ["\u0c3e"-"\u0c44"]
1208        | ["\u0c46"-"\u0c48"]
1209        | ["\u0c4a"-"\u0c4d"]
1210        | ["\u0c55"-"\u0c56"]
1211        | ["\u0c82"-"\u0c83"]
1212        | ["\u0cbe"-"\u0cc4"]
1213        | ["\u0cc6"-"\u0cc8"]
1214        | ["\u0cca"-"\u0ccd"]
1215        | ["\u0cd5"-"\u0cd6"]
1216        | ["\u0d02"-"\u0d03"]
1217        | ["\u0d3e"-"\u0d43"]
1218        | ["\u0d46"-"\u0d48"]
1219        | ["\u0d4a"-"\u0d4d"]
1220        | ["\u0d57"]
1221        | ["\u0d82"-"\u0d83"]
1222        | ["\u0dca"]
1223        | ["\u0dcf"-"\u0dd4"]
1224        | ["\u0dd6"]
1225        | ["\u0dd8"-"\u0ddf"]
1226        | ["\u0df2"-"\u0df3"]
1227        | ["\u0e31"]
1228        | ["\u0e34"-"\u0e3a"]
1229        | ["\u0e47"-"\u0e4e"]
1230        | ["\u0eb1"]
1231        | ["\u0eb4"-"\u0eb9"]
1232        | ["\u0ebb"-"\u0ebc"]
1233        | ["\u0ec8"-"\u0ecd"]
1234        | ["\u0f18"-"\u0f19"]
1235        | ["\u0f35"]
1236        | ["\u0f37"]
1237        | ["\u0f39"]
1238        | ["\u0f3e"-"\u0f3f"]
1239        | ["\u0f71"-"\u0f84"]
1240        | ["\u0f86"-"\u0f87"]
1241        | ["\u0f90"-"\u0f97"]
1242        | ["\u0f99"-"\u0fbc"]
1243        | ["\u0fc6"]
1244        | ["\u102c"-"\u1032"]
1245        | ["\u1036"-"\u1039"]
1246        | ["\u1056"-"\u1059"]
1247        | ["\u17b4"-"\u17d3"]
1248        | ["\u18a9"]
1249        | ["\u20d0"-"\u20dc"]
1250        | ["\u20e1"]
1251        | ["\u302a"-"\u302f"]
1252        | ["\u3099"-"\u309a"]
1253        | ["\ufb1e"]
1254        | ["\ufe20"-"\ufe23"]
1255    >
1256}
1257
1258TOKEN:
1259{
1260    < UNICODE_DIGIT:
1261        ["0"-"9"]
1262        | ["\u0660"-"\u0669"]
1263        | ["\u06f0"-"\u06f9"]
1264        | ["\u0966"-"\u096f"]
1265        | ["\u09e6"-"\u09ef"]
1266        | ["\u0a66"-"\u0a6f"]
1267        | ["\u0ae6"-"\u0aef"]
1268        | ["\u0b66"-"\u0b6f"]
1269        | ["\u0be7"-"\u0bef"]
1270        | ["\u0c66"-"\u0c6f"]
1271        | ["\u0ce6"-"\u0cef"]
1272        | ["\u0d66"-"\u0d6f"]
1273        | ["\u0e50"-"\u0e59"]
1274        | ["\u0ed0"-"\u0ed9"]
1275        | ["\u0f20"-"\u0f29"]
1276        | ["\u1040"-"\u1049"]
1277        | ["\u1369"-"\u1371"]
1278        | ["\u17e0"-"\u17e9"]
1279        | ["\u1810"-"\u1819"]
1280        | ["\uff10"-"\uff19"]
1281  >
1282}
1283
1284TOKEN:
1285{
1286    < UNICODE_CONNECTOR_PUNCTUATION:
1287        ["_"]
1288    |
1289        ["\u203f"-"\u2040"]
1290    |
1291        ["\u30fb"]
1292    |
1293        ["\ufe33"-"\ufe34"]
1294    |
1295        ["\ufe4d"-"\ufe4f"]
1296    |
1297        ["\uff3f"]
1298    |
1299        ["\uff65"]
1300    >
1301}
1302
1303TOKEN:
1304{
1305    < UNICODE_ESCAPE_SEQUENCE: "u" <HEX_DIGIT> <HEX_DIGIT> <HEX_DIGIT> <HEX_DIGIT> >
1306}
1307
1308TOKEN:
1309{
1310    < HEX_DIGIT: ["0"-"9"] | ["a"-"f"] | ["A"-"F"] >
1311}
1312
1313
1314
1315<IN_REGEX>
1316TOKEN :
1317{
1318    < SLASHASSIGN: "/=" > : DEFAULT
1319|
1320    < SLASH: "/" > : DEFAULT
1321}
1322
1323
1324/* Section 7.8.5: Regular Expression Literals */
1325
1326<DEFAULT>
1327TOKEN :
1328{
1329    < REGULAR_EXPRESSION_LITERAL:
1330        "/" ( (~["\n","\r","\\","/","*"]) | <BACKSLASH_SEQUENCE> )
1331        ( (~["\n","\r","\\","/"]) | <BACKSLASH_SEQUENCE> )* "/" (<IDENTIFIER_PART>)*
1332    > : IN_REGEX
1333|
1334    < #BACKSLASH_SEQUENCE:
1335        "\\" (~["\n","\r"])
1336    >
1337}
1338
1339
1340/*
1341
1342<DEFAULT>
1343TOKEN:
1344{
1345    < REGULAR_EXPRESSION_LITERAL:
1346        "/" <REGULAR_EXPRESSION_BODY> "/" <REGULAR_EXPRESSION_FLAGS>
1347    > : IN_REGEX
1348|
1349    < #REGULAR_EXPRESSION_BODY: <REGULAR_EXPRESSION_FIRSTCHAR> <REGULAR_EXPRESSION_CHARS> >
1350|
1351    < #REGULAR_EXPRESSION_CHARS: (<REGULAR_EXPRESSION_CHAR>)* >
1352|
1353    < #REGULAR_EXPRESSION_FIRSTCHAR:
1354    // ~["*","\\","/"] <NON_TERMINATOR>
1355    <NON_TERMINATOR_FIRSTCHAR>
1356    | <BACKSLASH_SEQUENCE>
1357    >
1358|
1359    < #REGULAR_EXPRESSION_CHAR:
1360        //~["\\","/"] <NON_TERMINATOR> <NON_TERMINATOR_CHAR>
1361        | <BACKSLASH_SEQUENCE>
1362    >
1363|
1364    < #BACKSLASH_SEQUENCE: "\\" <NON_TERMINATOR> >
1365|
1366    < #NON_TERMINATOR_FIRSTCHAR: ~["\n","\r","\u2028","\u2029","*","\\","/"] >
1367|
1368    < #NON_TERMINATOR_CHAR: ~["\n","\r","\u2028","\u2029","\\","/"] >
1369|
1370    < #NON_TERMINATOR: ~["\n","\r","\u2028","\u2029"] >
1371|
1372    < #REGULAR_EXPRESSION_FLAGS: (<IDENTIFIER_PART>)* >
1373}
1374
1375*/
1376
1377
1378/*****************************************
1379 *	NUMERIC STRING GRAMMAR	STARTS HERE	 *
1380 *****************************************/
1381
1382/* Section 9.3: String-to-Number Conversion */
1383
1384/*
1385MORE:
1386{
1387    <STRING_NUMERIC_LITERAL: (<STR_WHITESPACE>)? (<STR_NUMERIC_LITERAL> (<STR_WHITESPACE>)?)? >
1388}
1389
1390MORE:
1391{
1392    < STR_WHITESPACE: (<STR_WHITESPACE_CHAR>)+ >
1393}
1394
1395MORE:
1396{
1397    < STR_WHITESPACE_CHAR:
1398        <TAB>
1399        | <SP>
1400        | <NBSP>
1401        | <FF>
1402        | <VT>
1403        | <CR>
1404        | <LF>
1405        | <LS>
1406        | <PS>
1407        | <USP>
1408    >
1409}
1410
1411
1412MORE:
1413{
1414    < STR_NUMERIC_LITERAL:
1415        <STR_DECIMAL_LITERAL> | <HEX_INTEGER_LITERAL>
1416    >
1417}
1418
1419MORE:
1420{
1421    <STR_DECIMAL_LITERAL: <STR_UNSIGNED_DECIMAL_LITERAL> ("+" | "-") <STR_UNSIGNED_DECIMAL_LITERAL>	>
1422}
1423
1424MORE:
1425{
1426    < STR_UNSIGNED_DECIMAL_LITERAL:
1427    "Infinity"
1428    | <DECIMAL_DIGITS> "." (<DECIMAL_DIGITS>)* (<EXPONENT_PART>)*
1429    >
1430}
1431
1432*/
1433
1434
1435/* A.6 Universal Resource Identifier Character Classes */
1436
1437
1438/*
1439
1440TOKEN:
1441{
1442    < URIRESERVED : [";" , "/" , "?" , ":" , "@" , "&" , "=" , "+" , "$" , ","] >
1443}
1444
1445TOKEN:
1446{
1447    < URI_ALPHA : ["a"-"z","A"-"Z"] >
1448}
1449
1450TOKEN:
1451{
1452    < URI_MARK : ["-" , "_" , "." , "!" , "~" , "*" , "'" , "(" , ")"] >
1453}
1454
1455void uri() :
1456{}
1457{
1458    (uriCharacters())?
1459}
1460
1461void uriCharacters() :
1462{}
1463{
1464    (uriCharacter())+
1465}
1466
1467void uriCharacter() :
1468{}
1469{
1470    <URIRESERVED>
1471    | uriUnescaped()
1472    | uriEscaped()
1473}
1474
1475void uriUnescaped() :
1476{}
1477{
1478    <URI_ALPHA>
1479    | <DECIMAL_DIGIT>
1480    | <URI_MARK>
1481}
1482
1483void uriEscaped() :
1484{}
1485{
1486    "%" <HEX_DIGIT> <HEX_DIGIT>
1487}
1488
1489*/
1490
1491/* A.7 Regular Expressions */
1492
1493/*
1494
1495MORE :
1496{
1497    "/^" : INSIDE_PATTERN
1498}
1499
1500<INSIDE_PATTERN, IN_PATTERN>
1501TOKEN:
1502{
1503    < PATTERN : <DISJUNCTION> > : DEFAULT
1504|
1505    < #DISJUNCTION : (<ALTERNATIVE>)+ >
1506|
1507    < #ALTERNATIVE : (<TERM>)* >
1508|
1509    < #TERM : <ASSERTION> | <ATOM> (<QUANTIFIER>)? >
1510|
1511    < #ASSERTION :
1512        "^"
1513        | "$"
1514        | "\\" ("b" | "B")
1515    >
1516|
1517    < #QUANTIFIER : <QUANTIFIERPREFIX> ("?")? >
1518|
1519    < #QUANTIFIERPREFIX : "*" | "+" | "?" | ( "{" <DECIMAL_DIGITS> ("," (<DECIMAL_DIGITS>)?)? "}" ) >
1520|
1521    < #ATOM :
1522        <PATTERN_CHARACTER>
1523        | "."
1524        | "\\" <ATOMESCAPE>
1525        | <CHARACTERCLASS>
1526        //| "(" ("?" (":" | "=" | "!"))?	 <DISJUNCTION> ")"
1527        | "(" ("?" (":" | "=" | "!"))?  ")"
1528    >
1529|
1530    < #PATTERN_CHARACTER : ~["^", "$", "\\", ".", "*", "+", "?", "(", ")", "[", "]", "{", "}", "|"] >
1531|
1532    < #ATOMESCAPE : <DECIMALESCAPE> | <CHARACTERESCAPE> | <CHARACTER_CLASS_ESCAPE> >
1533|
1534    < #CHARACTER_CLASS_ESCAPE : ["d", "D", "s", "S", "w", "W"] >
1535|
1536    < #CHARACTERESCAPE :
1537        <CONTROL_ESCAPE>
1538        | "c" <CONTROL_LETTER>
1539        | <HEX_ESCAPE_SEQUENCE>
1540        | <UNICODE_ESCAPE_SEQUENCE>
1541        | <IDENTITY_ESCAPE>
1542    >
1543|
1544    < #CONTROL_ESCAPE : ["f" , "n" , "r" , "t" , "v"] >
1545|
1546    < #CONTROL_LETTER : ["a"-"z","A"-"Z"] >
1547|
1548    < #IDENTITY_ESCAPE : ~["0"-"9", "A"-"Z", "a"-"z","-"] >
1549|
1550    < #DECIMALESCAPE : <DECIMAL_INTEGER_LITERAL> >
1551|
1552    < #CHARACTERCLASS : "[" ("^")? <CLASSRANGES> "]" >
1553|
1554    < #CLASSRANGES : (<NONEMPTYCLASSRANGES>)? >
1555|
1556    //< #NONEMPTYCLASSRANGES : <CLASSATOM> (<NONEMPTYCLASSRANGESNODASH> | "-" <CLASSATOM> <CLASSRANGES>) >
1557    < #NONEMPTYCLASSRANGES : <CLASSATOM> (<NONEMPTYCLASSRANGESNODASH> | "-" <CLASSATOM>) >
1558|
1559    //< #NONEMPTYCLASSRANGESNODASH : <CLASSATOM> | <CLASSATOMNODASH> ( <NONEMPTYCLASSRANGESNODASH> | "-" <CLASSATOM> <CLASSRANGES> ) >
1560    < #NONEMPTYCLASSRANGESNODASH : <CLASSATOM> | <CLASSATOMNODASH> ( <CLASSATOM> | "-" <CLASSATOM> ) >
1561|
1562    < #CLASSATOM : "-" <CLASSATOMNODASH> >
1563|
1564    < #CLASSATOMNODASH : <CLASS_ATOM_NODASH_TOKEN> | "\\" <CLASSESCAPE> >
1565|
1566    < #CLASS_ATOM_NODASH_TOKEN : ~["\\", "]", "-"] >
1567|
1568    < #CLASSESCAPE :
1569        <DECIMALESCAPE>
1570        | "b"
1571        | <CHARACTERESCAPE>
1572        | <CHARACTER_CLASS_ESCAPE>
1573    >
1574}
1575
1576*/
1577
1578/*****************************************
1579 * ECMA SYNTACTIC GRAMMARS  STARTS HERE  *
1580 *****************************************/
1581
1582    /* Section 11.1: Primary Expressions */
1583
1584    void PrimaryExpression()       :
1585    {}
1586    {
1587    try {
1588            LOOKAHEAD("this")/*@bgen(jjtree) ThisReference */
1589                              {
1590                                ASTThisReference jjtn001 = new ASTThisReference(JJTTHISREFERENCE);
1591                                boolean jjtc001 = true;
1592                                jjtree.openNodeScope(jjtn001);
1593                                jjtreeOpenNodeScope(jjtn001);
1594                              }
1595                              try {
1596/*@egen*/ "this"/*@bgen(jjtree)*/
1597                              } finally {
1598                                if (jjtc001) {
1599                                  jjtree.closeNodeScope(jjtn001, true);
1600                                  jjtreeCloseNodeScope(jjtn001);
1601                                }
1602                              }
1603/*@egen*/
1604        | LOOKAHEAD("{") ObjectLiteral()
1605        | LOOKAHEAD("(")/*@bgen(jjtree) ParenExpression */
1606                         {
1607                           ASTParenExpression jjtn002 = new ASTParenExpression(JJTPARENEXPRESSION);
1608                           boolean jjtc002 = true;
1609                           jjtree.openNodeScope(jjtn002);
1610                           jjtreeOpenNodeScope(jjtn002);
1611                         }
1612                         try {
1613/*@egen*/ ( "(" Expression() ")" )/*@bgen(jjtree)*/
1614                         } catch (Throwable jjte002) {
1615                           if (jjtc002) {
1616                             jjtree.clearNodeScope(jjtn002);
1617                             jjtc002 = false;
1618                           } else {
1619                             jjtree.popNode();
1620                           }
1621                           if (jjte002 instanceof RuntimeException) {
1622                             throw (RuntimeException)jjte002;
1623                           }
1624                           if (jjte002 instanceof ParseException) {
1625                             throw (ParseException)jjte002;
1626                           }
1627                           throw (Error)jjte002;
1628                         } finally {
1629                           if (jjtc002) {
1630                             jjtree.closeNodeScope(jjtn002, true);
1631                             jjtreeCloseNodeScope(jjtn002);
1632                           }
1633                         }
1634/*@egen*/
1635        | LOOKAHEAD(Identifier()) Identifier()
1636        | LOOKAHEAD("[") ArrayLiteral()
1637        | LOOKAHEAD(2) Literal()
1638    }
1639        catch(ParseException pe) {
1640            addException(pe);
1641        }
1642        }
1643
1644    /* Section 7.8: Literals */
1645
1646    void Literal()          :
1647    {/*@bgen(jjtree) Literal */
1648      ASTLiteral jjtn000 = new ASTLiteral(JJTLITERAL);
1649      boolean jjtc000 = true;
1650      jjtree.openNodeScope(jjtn000);
1651      jjtreeOpenNodeScope(jjtn000);
1652/*@egen*/
1653      Token t;
1654      Map objLiteral;
1655      List arrayLiteral;
1656    }
1657    {/*@bgen(jjtree) Literal */
1658    try {
1659/*@egen*/
1660    try {
1661            t = <DECIMAL_LITERAL>/*@bgen(jjtree)*/
1662        {
1663          jjtree.closeNodeScope(jjtn000, true);
1664          jjtc000 = false;
1665          jjtreeCloseNodeScope(jjtn000);
1666        }
1667/*@egen*/
1668        {
1669            jjtn000.setDecimalValue(t.image);
1670        }
1671    |
1672        t = <HEX_INTEGER_LITERAL>/*@bgen(jjtree)*/
1673        {
1674          jjtree.closeNodeScope(jjtn000, true);
1675          jjtc000 = false;
1676          jjtreeCloseNodeScope(jjtn000);
1677        }
1678/*@egen*/
1679        {
1680            jjtn000.setHexValue(t.image);
1681        }
1682    |
1683        t = <STRING_LITERAL>/*@bgen(jjtree)*/
1684        {
1685          jjtree.closeNodeScope(jjtn000, true);
1686          jjtc000 = false;
1687          jjtreeCloseNodeScope(jjtn000);
1688        }
1689/*@egen*/
1690        {
1691            jjtn000.setStringValue(t.image);
1692        }
1693    |
1694        t = <BOOLEAN_LITERAL>/*@bgen(jjtree)*/
1695        {
1696          jjtree.closeNodeScope(jjtn000, true);
1697          jjtc000 = false;
1698          jjtreeCloseNodeScope(jjtn000);
1699        }
1700/*@egen*/
1701        {
1702            jjtn000.setBooleanValue(t.image);
1703        }
1704    |
1705        t = <NULL_LITERAL>/*@bgen(jjtree)*/
1706        {
1707          jjtree.closeNodeScope(jjtn000, true);
1708          jjtc000 = false;
1709          jjtreeCloseNodeScope(jjtn000);
1710        }
1711/*@egen*/
1712        {
1713            jjtn000.setNullValue();
1714        }
1715    |
1716        t = <REGULAR_EXPRESSION_LITERAL>/*@bgen(jjtree)*/
1717        {
1718          jjtree.closeNodeScope(jjtn000, true);
1719          jjtc000 = false;
1720          jjtreeCloseNodeScope(jjtn000);
1721        }
1722/*@egen*/
1723        {
1724            jjtn000.setRegexValue(t.image);
1725        }
1726    }
1727        catch(ParseException pe) {
1728            addException(pe);
1729        }/*@bgen(jjtree)*/
1730    } finally {
1731      if (jjtc000) {
1732        jjtree.closeNodeScope(jjtn000, true);
1733        jjtreeCloseNodeScope(jjtn000);
1734      }
1735    }
1736/*@egen*/
1737        }
1738
1739
1740    void Identifier()             :
1741    {/*@bgen(jjtree) Identifier */
1742      ASTIdentifier jjtn000 = new ASTIdentifier(JJTIDENTIFIER);
1743      boolean jjtc000 = true;
1744      jjtree.openNodeScope(jjtn000);
1745      jjtreeOpenNodeScope(jjtn000);
1746/*@egen*/
1747      Token t;
1748    }
1749    {/*@bgen(jjtree) Identifier */
1750    try {
1751/*@egen*/
1752    try {
1753          t=<IDENTIFIER_NAME>/*@bgen(jjtree)*/
1754      {
1755        jjtree.closeNodeScope(jjtn000, true);
1756        jjtc000 = false;
1757        jjtreeCloseNodeScope(jjtn000);
1758      }
1759/*@egen*/
1760      {
1761        jjtn000.setName(t.image);
1762      }
1763    }
1764        catch(ParseException pe) {
1765            addException(pe);
1766        }/*@bgen(jjtree)*/
1767    } finally {
1768      if (jjtc000) {
1769        jjtree.closeNodeScope(jjtn000, true);
1770        jjtreeCloseNodeScope(jjtn000);
1771      }
1772    }
1773/*@egen*/
1774        }
1775
1776    /* Section 11.1.4: Array Initialiser */
1777
1778    void ArrayLiteral()               :
1779    {/*@bgen(jjtree) ArrayLiteral */
1780  ASTArrayLiteral jjtn000 = new ASTArrayLiteral(JJTARRAYLITERAL);
1781  boolean jjtc000 = true;
1782  jjtree.openNodeScope(jjtn000);
1783  jjtreeOpenNodeScope(jjtn000);
1784/*@egen*/}
1785    {/*@bgen(jjtree) ArrayLiteral */
1786    try {
1787/*@egen*/
1788    try {
1789            "[" (
1790                LOOKAHEAD(2) ( Elision() )? "]"
1791                        | LOOKAHEAD(ElementList() Elision()) ElementList() Elision() "]"
1792                        | ( ElementList() )? "]"
1793        )
1794    }
1795        catch(ParseException pe) {
1796            addException(pe);
1797        }/*@bgen(jjtree)*/
1798    } catch (Throwable jjte000) {
1799      if (jjtc000) {
1800        jjtree.clearNodeScope(jjtn000);
1801        jjtc000 = false;
1802      } else {
1803        jjtree.popNode();
1804      }
1805      if (jjte000 instanceof RuntimeException) {
1806        throw (RuntimeException)jjte000;
1807      }
1808      if (jjte000 instanceof ParseException) {
1809        throw (ParseException)jjte000;
1810      }
1811      throw (Error)jjte000;
1812    } finally {
1813      if (jjtc000) {
1814        jjtree.closeNodeScope(jjtn000, true);
1815        jjtreeCloseNodeScope(jjtn000);
1816      }
1817    }
1818/*@egen*/
1819        }
1820
1821    void ElementList()       :
1822    {}
1823    {
1824    try {
1825            ( Elision() )? AssignmentExpression() ( LOOKAHEAD(Elision()	AssignmentExpression()) Elision()  AssignmentExpression())*
1826    }
1827        catch(ParseException pe) {
1828            addException(pe);
1829        }
1830        }
1831
1832    void Elision()       :
1833    {}
1834    {
1835    try {
1836            (",")+
1837    }
1838        catch(ParseException pe) {
1839            addException(pe);
1840        }
1841        }
1842
1843    /* Section 11.1.5: Object Initialiser */
1844
1845    void ObjectLiteral()                :
1846    {/*@bgen(jjtree) ObjectLiteral */
1847  ASTObjectLiteral jjtn000 = new ASTObjectLiteral(JJTOBJECTLITERAL);
1848  boolean jjtc000 = true;
1849  jjtree.openNodeScope(jjtn000);
1850  jjtreeOpenNodeScope(jjtn000);
1851/*@egen*/}
1852    {/*@bgen(jjtree) ObjectLiteral */
1853    try {
1854/*@egen*/
1855    try {
1856            "{" ( PropertyNameAndValueList() )? "}"
1857    }
1858        catch(ParseException pe) {
1859            addException(pe);
1860        }/*@bgen(jjtree)*/
1861    } catch (Throwable jjte000) {
1862      if (jjtc000) {
1863        jjtree.clearNodeScope(jjtn000);
1864        jjtc000 = false;
1865      } else {
1866        jjtree.popNode();
1867      }
1868      if (jjte000 instanceof RuntimeException) {
1869        throw (RuntimeException)jjte000;
1870      }
1871      if (jjte000 instanceof ParseException) {
1872        throw (ParseException)jjte000;
1873      }
1874      throw (Error)jjte000;
1875    } finally {
1876      if (jjtc000) {
1877        jjtree.closeNodeScope(jjtn000, true);
1878        jjtreeCloseNodeScope(jjtn000);
1879      }
1880    }
1881/*@egen*/
1882        }
1883
1884    void PropertyNameAndValueList()       :
1885    {}
1886    {
1887    try {
1888            PropertyNameAndValue() ( LOOKAHEAD( "," PropertyNameAndValue()) "," PropertyNameAndValue() | "," )*
1889    }
1890        catch(ParseException pe) {
1891            addException(pe);
1892        }
1893        }
1894
1895    void PropertyNameAndValue()               :
1896    {/*@bgen(jjtree) LiteralField */
1897  ASTLiteralField jjtn000 = new ASTLiteralField(JJTLITERALFIELD);
1898  boolean jjtc000 = true;
1899  jjtree.openNodeScope(jjtn000);
1900  jjtreeOpenNodeScope(jjtn000);
1901/*@egen*/}
1902    {/*@bgen(jjtree) LiteralField */
1903        try {
1904/*@egen*/
1905        try {
1906            PropertyName() ":" (LOOKAHEAD("function") FunctionExpression() | AssignmentExpression())
1907        }
1908        catch(ParseException pe) {
1909            addException(pe);
1910        }/*@bgen(jjtree)*/
1911        } catch (Throwable jjte000) {
1912          if (jjtc000) {
1913            jjtree.clearNodeScope(jjtn000);
1914            jjtc000 = false;
1915          } else {
1916            jjtree.popNode();
1917          }
1918          if (jjte000 instanceof RuntimeException) {
1919            throw (RuntimeException)jjte000;
1920          }
1921          if (jjte000 instanceof ParseException) {
1922            throw (ParseException)jjte000;
1923          }
1924          throw (Error)jjte000;
1925        } finally {
1926          if (jjtc000) {
1927            jjtree.closeNodeScope(jjtn000, true);
1928            jjtreeCloseNodeScope(jjtn000);
1929          }
1930        }
1931/*@egen*/
1932    }
1933
1934    void PropertyName()       :
1935    {}
1936    {
1937    try {
1938            Identifier()
1939        |/*@bgen(jjtree) Literal */
1940        {
1941          ASTLiteral jjtn001 = new ASTLiteral(JJTLITERAL);
1942          boolean jjtc001 = true;
1943          jjtree.openNodeScope(jjtn001);
1944          jjtreeOpenNodeScope(jjtn001);
1945        }
1946        try {
1947/*@egen*/
1948        <STRING_LITERAL>/*@bgen(jjtree)*/
1949        } finally {
1950          if (jjtc001) {
1951            jjtree.closeNodeScope(jjtn001, true);
1952            jjtreeCloseNodeScope(jjtn001);
1953          }
1954        }
1955/*@egen*/
1956        |/*@bgen(jjtree) Literal */
1957        {
1958          ASTLiteral jjtn002 = new ASTLiteral(JJTLITERAL);
1959          boolean jjtc002 = true;
1960          jjtree.openNodeScope(jjtn002);
1961          jjtreeOpenNodeScope(jjtn002);
1962        }
1963        try {
1964/*@egen*/
1965        <DECIMAL_LITERAL>/*@bgen(jjtree)*/
1966        } finally {
1967          if (jjtc002) {
1968            jjtree.closeNodeScope(jjtn002, true);
1969            jjtreeCloseNodeScope(jjtn002);
1970          }
1971        }
1972/*@egen*/
1973    }
1974        catch(ParseException pe) {
1975            addException(pe);
1976        }
1977        }
1978
1979
1980    /* Section 11.2: Left-Hand-Side Expressions */
1981
1982    void MemberExpression()       :
1983    {}
1984    {
1985    try {/*@bgen(jjtree) #CompositeReference(> 1) */
1986            {
1987              ASTCompositeReference jjtn001 = new ASTCompositeReference(JJTCOMPOSITEREFERENCE);
1988              boolean jjtc001 = true;
1989              jjtree.openNodeScope(jjtn001);
1990              jjtreeOpenNodeScope(jjtn001);
1991            }
1992            try {
1993/*@egen*/
1994            ( (
1995                LOOKAHEAD("function") FunctionExpression()
1996                | PrimaryExpression()
1997        ) (LOOKAHEAD(2) MemberExpressionPart())*	 )/*@bgen(jjtree)*/
1998            } catch (Throwable jjte001) {
1999              if (jjtc001) {
2000                jjtree.clearNodeScope(jjtn001);
2001                jjtc001 = false;
2002              } else {
2003                jjtree.popNode();
2004              }
2005              if (jjte001 instanceof RuntimeException) {
2006                throw (RuntimeException)jjte001;
2007              }
2008              if (jjte001 instanceof ParseException) {
2009                throw (ParseException)jjte001;
2010 

Large files files are truncated, but you can click here to view the full file