/tests/output/d/40007-Lexer.d

/+

 *      Copyright (c) 1999-2006 by Digital Mars

 *      All Rights Reserved

 *      written by Walter Bright www.digitalmars.com

 *      License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt.

 *      See the included readme.txt for details.

 *      D Language conversion by: J Duncan

 +/



/**

 *      d language lexer

 */



module dparser.Lexer;



import dparser.Root;



import dparser.Tokens;

import dparser.Token;

import dparser.Keyword;



import dparser.Types;



import dparser.Module;

import dparser.Identifier;

import dparser.unialpha;



import dparser.OutBuffer;



//private import std.ctype;

//private import std.string;

//import dwf.core.debugapi;



int errno = 0;



//#if _WIN32 && __DMC__

// from \dm\src\include\setlocal.h

//extern "C" char * __cdecl __locale_decpoint;

char* __locale_decpoint;

//#endif

//const uint LS = 0x2028;	// UTF line separator

//const uint PS = 0x2029;	// UTF paragraph separator



//extern int isUniAlpha(unsigned u);

//extern int HtmlNamedEntity(unsigned char *p, int length);



/**

 *      Lexer object

 */



class Lexer

{

    static           Identifier[char[]]       stringtable;

    static OutBuffer stringbuffer;

    static Token     * freelist;



    Token            token;        // current token

    Module           mod;          // current module

    Loc              loc;          // for error messages

    ubyte            *base;        // pointer to start of buffer

    ubyte            *end;         // past end of buffer

    ubyte            *p;           // current character

    int              doDocComment; // collect doc comment information

    int              anyToken;     // !=0 means seen at least one token

    int              commentToken; // !=0 means comments are TOKcomment's





    this(Module mod, ubyte* base, uint begoffset, uint endoffset, int doDocComment, int commentToken)

    {

        if (stringbuffer is null) {

            stringbuffer = new OutBuffer;

        }

        loc = Loc(mod, 1);



        this.base         = base;

        this.end          = base + endoffset;

        this.p            = base + begoffset;

        this.mod          = mod;

        this.doDocComment = doDocComment;

        this.commentToken = commentToken;



        /*

         *      If first line starts with '#!', ignore the line

         */



        if (p[0] == '#' && p[1] == '!') {

            p += 2;

            while (true) {

                ubyte c = *p;

                switch (c) {

                case '\n':

                    p++;

                    break;



                case '\r':

                    p++;

                    if (*p == '\n') {

                        p++;

                    }

                    break;



                case 0:

                case 0x1A:

                    break;



                default:

                    if (c & 0x80) {

                        uint u = decodeUTF();

                        if (u == PS || u == LS) {

                            break;

                        }

                    }

                    p++;

                    continue;

                }

                break;

            }



            loc.linnum = 2;

        }

    }







    // generate a unique identifier for this string

    static Identifier idPool(in char[] str) {

//	    StringValue sv;

//	    uint len = s.length;

//	    StringValue sv = stringtable.update(s, len);

//	    Identifier* id = cast(Identifier*) sv.ptrvalue;

//	    if( id is null )

        if ((str in stringtable) == null) {

            stringtable[str] = new Identifier(str, TOK.TOKidentifier);

        }

        return(stringtable[str]);

    }



    static void initKeywords() {

        // build character map

        cmtable_init();



        // create keyword tokens & identifiers

        dparser.Keyword.initKeywords();



        // create standard lexer tokens

        dparser.Token.createLexerTokens();

    }



    // Combine two document comments into one.

    static char[] combineComments(char[] c1, char[] c2) {

        char[] c = c2;

        if (c1.length) {

            c = c1;

            if (c2.length) {

                c = c1 ~ "\n" ~ c2;

            }

        }

        return(c);

    }



    // Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence.

    //! fix

    uint decodeUTF() {

        ubyte * s = p;

        ubyte c   = *s;



        assert(c & 0x80);

        if (!(c & 0x80)) {

            return(c);

        }



        return(cast(uint)'X');

        /*

         *  dchar u;

         *  uint len;

         *

         *

         *

         *  // Check length of remaining string up to 6 UTF-8 characters

         *  for( len = 1; len < 6 && s[len]; len++ )

         *  {

         *

         *  }

         *              /+

         *  uint idx = 0;

         *  char* msg = utf_decodeChar( s, len, &idx, &u );

         *  p += idx - 1;

         *  if( msg )

         *  {

         *              error(msg);

         *  }

         * +/

         *  return u;

         */

    }



    void error(...) {

        if ((mod !is null) && !global.gag) {

            writefln(formatLoc(loc, _arguments, _argptr));

            /*

             * char[] p = loc.toChars();

             * if( p.length )

             *  writef( "%s: ", p );

             * writefx( stdout, _arguments, _argptr, 1 );

             */

            if (global.errors >= global.max_errors) {                   // moderate blizzard of cascading messages

                throw new Exception("too many errors");

            }

        }



        global.errors++;

    }



    void errorLoc(Loc loc, ...) {

        if ((mod !is null) && !global.gag) {

            writefln(formatLoc(loc, _arguments, _argptr));

            /*

             * char[] p = loc.toChars();

             * if( p.length )

             *  writef("%s: ", p);

             * writefx(stdout, _arguments, _argptr, 1);

             */

            if (global.errors >= 20) {                  // moderate blizzard of cascading messages

                throw new Exception("too many errors");

            }

        }



        global.errors++;

    }





    TOK nextToken() {

        if (token.next) {

            Token* t = token.next;

            memcpy(&token, t, Token.sizeof);

//			t.next = freelist;

//			freelist = t;

        }

        else {

            scan(&token);

        }

//	    token.print();

        return(token.value);

    }



    Token* peek(inout Token ct) {

        Token* t;



        if (ct.next) {

            t = ct.next;

        }

        else {

            t = new Token;

            scan(t);

            t.next  = null;

            ct.next = t;

        }

        return(t);

    }



    // Turn next token in buffer into a token.



    void scan(Token* t) {

//		debug writefln("scan token");

        uint lastLine = loc.linnum;

        uint linnum;



        t.blockComment = null;

        t.lineComment  = null;

        while (true) {

            t.ptr = p;

//			debug writefln( "    p = %d, *p = ", cast(uint)p, cast(char)*p );

            switch (*p) {

            case 0:

            case 0x1a:

                t.value = TOK.TOKeof;                                           // end of file

//					debug writefln( "    EOF" );

                return;



            case ' ':

            case '\t':

            case '\v':

            case '\f':

                p++;

//					debug writefln( "    whitespace" );

                continue;                                                               // skip white space



            case '\r':

//					debug writefln( "    cr" );

                p++;

                if (*p != '\n') {                                               // if CR stands by itself

                    loc.linnum++;

                }

                continue;                                                               // skip white space



            case '\n':

//					debug writefln( "    nl" );

                p++;

                loc.linnum++;

                continue;                                                               // skip white space



            case '0':

            case '1':

            case '2':

            case '3':

            case '4':

            case '5':

            case '6':

            case '7':

            case '8':

            case '9':

                t.value = number(t);

                return;



/*

 * #if CSTRINGS

 *                          case '\'':

 *                              t.value = charConstant(t, 0);

 *                              return;

 *

 *                          case '"':

 *                              t.value = stringConstant(t,0);

 *                              return;

 *

 *                          case 'l':

 *                          case 'L':

 *                              if( p[1] == '\'')

 *                              {

 *                                  p++;

 *                                  t.value = charConstant(t, 1);

 *                                  return;

 *                              }

 *                              else if( p[1] == '"')

 *                              {

 *                                  p++;

 *                                  t.value = stringConstant(t, 1);

 *                                  return;

 *                              }

 * #else

 */

            case '\'':

//					debug writefln( "    char" );

                t.value = charConstant(t, 0);

                return;



            case 'r':

//					debug writefln( "    wysiwyg" );

                if (p[1] != '"') {

                    goto case_ident;

                }

                p++;



            case '`':

                t.value = wysiwygStringConstant(t, *p);

                return;



            case 'x':

//					debug writefln( "    hex string" );

                if (p[1] != '"') {

                    goto case_ident;

                }

                p++;

                t.value = hexStringConstant(t);

                return;





            case '"':

//					debug writefln( "    string" );

                t.value = escapeStringConstant(t, 0);

//					debug writefln( t.ustring );

                return;



            case '\\':                                  // escaped string literal

//					debug writefln( "    escaped string literal" );

                uint c;

                stringbuffer.offset = 0;

                do {

                    p++;

                    c = escapeSequence();

                    stringbuffer.write(c);

                } while (*p == '\\');

//					t.len = stringbuffer.offset;

//					stringbuffer.write(cast(byte)0);

                t.ustring = stringbuffer.toString;

//					memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset );

                t.postfix = 0;

                t.value   = TOK.TOKstring;

                return;



            case 'l':

            case 'L':

//	#endif



            case 'a':

            case 'b':

            case 'c':

            case 'd':

            case 'e':

            case 'f':

            case 'g':

            case 'h':

            case 'i':

            case 'j':

            case 'k':

            case 'm':

            case 'n':

            case 'o':

            case 'p':

            case 'q':                             /*case 'r':*/

            case 's':

            case 't':

            case 'u':

            case 'v':

            case 'w':                                         /*case 'x':*/

            case 'y':

            case 'z':

            case 'A':

            case 'B':

            case 'C':

            case 'D':

            case 'E':

            case 'F':

            case 'G':

            case 'H':

            case 'I':

            case 'J':

            case 'K':

            case 'M':

            case 'N':

            case 'O':

            case 'P':

            case 'Q':

            case 'R':

            case 'S':

            case 'T':

            case 'U':

            case 'V':

            case 'W':

            case 'X':

            case 'Y':

            case 'Z':

            case '_':

case_ident:

                {

//					debug writefln( "    identifier" );

                    ubyte c;

                    do {

                        c = *++p;

                    } while (isidchar(c) || (c & 0x80 && isUniAlpha(decodeUTF())));



//					sv = stringtable.update((char *)t.ptr, p - t.ptr);

                    char[] tmp;

                    tmp.length = p - t.ptr;

                    memcpy(tmp.ptr, t.ptr, p - t.ptr);

                    Identifier id;

                    Identifier * pid = tmp in stringtable;

                    if (pid) {

                        id = *pid;

                    }



                    if (id is null) {

                        id               = new Identifier(tmp, TOK.TOKidentifier);

                        stringtable[tmp] = id;

                    }



                    t.ident  = id;

                    t.value  = cast(TOK)id.value;

                    anyToken = 1;



                    // if special identifier token

                    if (*t.ptr == '_') {

                        static char date[11 + 1];

                        static char time[8 + 1];

                        static char timestamp[24 + 1];



                        if (!date[0]) {                         // lazy evaluation

                            //!!

                            /+

                             *      time_t t;

                             *      char *p;

                             *      .time(&t);

                             *      p = ctime(&t);

                             *      assert(p);

                             *      sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20);

                             *      sprintf(time.ptr, "%.8s", p + 11);

                             *      sprintf(timestamp.ptr, "%.24s", p);

                             +/

                        }



                        if (mod && id is Id.FILE) {

                            t.value = TOK.TOKstring;

                            if (loc.filename.length) {

                                t.ustring = loc.filename;

                            }

                            else {

                                t.ustring = mod.ident.toChars();

                            }

                            goto Llen;

                        }

                        else if (mod && id == Id.LINE) {

                            t.value      = TOK.TOKint64v;

                            t.uns64value = loc.linnum;

                        }

                        else if (id == Id.DATE) {

                            t.value = TOK.TOKstring;

                            //! t.ustring = date;

                            goto Llen;

                        }

                        else if (id == Id.TIME) {

                            t.value = TOK.TOKstring;

                            //! t.ustring = time;

                            goto Llen;

                        }

                        else if (id == Id.TIMESTAMP) {

                            t.value = TOK.TOKstring;

                            //! t.ustring = timestamp;

Llen:

                            t.postfix = 0;

//							t.len = strlen((char *)t.ustring);

                        }

                    }

                    //printf("t.value = %d\n",t.value);

                    return;

                }



            // comments

            case '/':

                p++;

                switch (*p) {

                case '=':

                    p++;

                    t.value = TOK.TOKdivass;

                    return;



                case '*':                               // '/*'

                    p++;

                    linnum = loc.linnum;

                    while (true) {

                        while (true) {

                            ubyte c = *p;

                            switch (c) {

                            case '/':

                                break;



                            case '\n':

                                loc.linnum++;

                                p++;

                                continue;



                            case '\r':

                                p++;

                                if (*p != '\n') {

                                    loc.linnum++;

                                }

                                continue;



                            case 0:

                            case 0x1A:

                                error("unterminated /* */ comment");

                                p       = end;

                                t.value = TOK.TOKeof;

                                return;



                            default:

                                if (c & 0x80) {

                                    uint u = decodeUTF();

                                    if (u == PS || u == LS) {

                                        loc.linnum++;

                                    }

                                }

                                p++;

                                continue;

                            }

                            break;

                        }

                        p++;

                        if (p[-2] == '*' && p - 3 != t.ptr) {

                            break;

                        }

                    }



                    if (commentToken) {

                        t.value = TOK.TOKcomment;

                        return;

                    }

                    // if /** but not /**/

                    else if (doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr) {

                        getDocComment(t, lastLine == linnum);                                           //! ?

                    }

                    continue;



                case '/':                                       // do // style comments

                    linnum = loc.linnum;

                    while (1) {

                        ubyte c = *++p;

                        switch (c) {

                        case '\n':

                            break;



                        case '\r':

                            if (p[1] == '\n') {

                                p++;

                            }

                            break;



                        case 0:

                        case 0x1a:

                            if (commentToken) {

                                p       = end;

                                t.value = TOK.TOKcomment;

                                return;

                            }

                            if (doDocComment && t.ptr[2] == '/') {

                                getDocComment(t, lastLine == linnum);

                            }

                            p       = end;

                            t.value = TOK.TOKeof;

                            return;



                        default:

                            if (c & 0x80) {

                                uint u = decodeUTF();

                                if (u == PS || u == LS) {

                                    break;

                                }

                            }

                            continue;

                        }

                        break;

                    }



                    if (commentToken) {

                        p++;

                        loc.linnum++;

                        t.value = TOK.TOKcomment;

                        return;

                    }

                    if (doDocComment && t.ptr[2] == '/') {

                        getDocComment(t, lastLine == linnum);

                    }



                    p++;

                    loc.linnum++;

                    continue;



                case '+':

                    {

                        int nest;

                        linnum = loc.linnum;

                        p++;

                        nest = 1;

                        while (1) {

                            ubyte c = *p;

                            switch (c) {

                            case '/':

                                p++;

                                if (*p == '+') {

                                    p++;

                                    nest++;

                                }

                                continue;



                            case '+':

                                p++;

                                if (*p == '/') {

                                    p++;

                                    if (--nest == 0) {

                                        break;

                                    }

                                }

                                continue;



                            case '\r':

                                p++;

                                if (*p != '\n') {

                                    loc.linnum++;

                                }

                                continue;



                            case '\n':

                                loc.linnum++;

                                p++;

                                continue;



                            case 0:

                            case 0x1A:

                                error("unterminated /+ +/ comment");

                                p       = end;

                                t.value = TOK.TOKeof;

                                return;



                            default:

                                if (c & 0x80) {

                                    uint u = decodeUTF();

                                    if (u == PS || u == LS) {

                                        loc.linnum++;

                                    }

                                }

                                p++;

                                continue;

                            }

                            break;

                        }

                        if (commentToken) {

                            t.value = TOK.TOKcomment;

                            return;

                        }

                        if (doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr) {

                            // if /++ but not /++/

                            getDocComment(t, lastLine == linnum);

                        }

                        continue;

                    }



                default:

                    break;

                }

                t.value = TOK.TOKdiv;

                return;



            case '.':

                p++;

                if (isdigit(*p)) {

                    p--;

                    t.value = inreal(t);

                }

                else if (p[0] == '.') {

                    if (p[1] == '.') {

                        p      += 2;

                        t.value = TOK.TOKdotdotdot;

                    }

                    else {

                        p++;

                        t.value = TOK.TOKslice;

                    }

                }

                else {

                    t.value = TOK.TOKdot;

                }

                return;



            case '&':

                p++;

                if (*p == '=') {

                    p++;

                    t.value = TOK.TOKandass;

                }

                else if (*p == '&') {

                    p++;

                    t.value = TOK.TOKandand;

                }

                else {

                    t.value = TOK.TOKand;

                }

                return;



            // |, ||, |=

            case '|':

                p++;

                if (*p == '=') {

                    p++;

                    t.value = TOK.TOKorass;

                }

                else if (*p == '|') {

                    p++;

                    t.value = TOK.TOKoror;

                }

                else {

                    t.value = TOK.TOKor;

                }

                return;



            case '-':

                p++;

                if (*p == '=') {

                    p++;

                    t.value = TOK.TOKminass;

                }

                else if (*p == '-') {

                    p++;

                    t.value = TOK.TOKminusminus;

                }

                else {

                    t.value = TOK.TOKmin;

                }

                return;



            // +, +=, ++

            case '+':

                p++;

                if (*p == '=') {

                    p++;

                    t.value = TOK.TOKaddass;                                    // +=

                }

                else if (*p == '+') {

                    p++;

                    t.value = TOK.TOKplusplus;                                          // ++

                }

                else {

                    t.value = TOK.TOKadd;                                                       // +

                }

                return;



            // <, <=, <<=, <<, <>=, <>

            case '<':

                p++;

                if (*p == '=') {

                    p++;

                    t.value = TOK.TOKle;                                                // <=

                }

                else if (*p == '<') {

                    p++;

                    if (*p == '=') {

                        p++;

                        t.value = TOK.TOKshlass;                                        // <<=

                    }

                    else {

                        t.value = TOK.TOKshl;                                           // <<

                    }

                }

                else if (*p == '>') {

                    p++;

                    if (*p == '=') {

                        p++;

                        t.value = TOK.TOKleg;                                           // <>=

                    }

                    else {

                        t.value = TOK.TOKlg;                                            // <>

                    }

                }

                else {

                    t.value = TOK.TOKlt;                                                // <

                }

                return;



            // >, >>, >>>, >=, >>=, >>>=

            case '>':

                p++;

                if (*p == '=') {

                    p++;

                    t.value = TOK.TOKge;                                                // >=

                }

                else if (*p == '>') {

                    p++;

                    if (*p == '=') {

                        p++;

                        t.value = TOK.TOKshrass;                                        // >>=

                    }

                    else if (*p == '>') {

                        p++;

                        if (*p == '=') {

                            p++;

                            t.value = TOK.TOKushrass;                           // >>>=

                        }

                        else {

                            t.value = TOK.TOKushr;                                      // >>>

                        }

                    }

                    else {

                        t.value = TOK.TOKshr;                                           // >>

                    }

                }

                else {

                    t.value = TOK.TOKgt;                                                // >

                }

                return;



            case '!':

                p++;

                if (*p == '=') {

                    p++;

                    if (*p == '=') {

                        p++;

                        t.value = TOK.TOKnotidentity;                                   // !==

                    }

                    else {

                        t.value = TOK.TOKnotequal;                                      // !=

                    }

                }

                else if (*p == '<') {

                    p++;

                    if (*p == '>') {

                        p++;

                        if (*p == '=') {

                            p++;

                            t.value = TOK.TOKunord;                             // !<>=

                        }

                        else {

                            t.value = TOK.TOKue;                                // !<>

                        }

                    }

                    else if (*p == '=') {

                        p++;

                        t.value = TOK.TOKug;                                    // !<=

                    }

                    else {

                        t.value = TOK.TOKuge;                                   // !<

                    }

                }

                else if (*p == '>') {

                    p++;

                    if (*p == '=') {

                        p++;

                        t.value = TOK.TOKul;                                    // !>=

                    }

                    else {

                        t.value = TOK.TOKule;                                   // !>

                    }

                }

                else {

                    t.value = TOK.TOKnot;                                       // !

                }

                return;



            case '=':

                p++;

                if (*p == '=') {

                    p++;

                    if (*p == '=') {

                        p++;

                        t.value = TOK.TOKidentity;                                      // ===

                    }

                    else {

                        t.value = TOK.TOKequal;                                         // ==

                    }

                }

                else {

                    t.value = TOK.TOKassign;                                    // =

                }

                return;



            case '~':

                p++;

                if (*p == '=') {

                    p++;

                    t.value = TOK.TOKcatass;                                            // ~=

                }

                else {

                    t.value = TOK.TOKtilde;                                             // ~

                }

                return;



            // SINGLE

            case '(': p++; t.value = TOK.TOKlparen;     return;



            case ')': p++; t.value = TOK.TOKrparen;     return;



            case '[': p++; t.value = TOK.TOKlbracket;   return;



            case ']': p++; t.value = TOK.TOKrbracket;   return;



            case '{': p++; t.value = TOK.TOKlcurly;     return;



            case '}': p++; t.value = TOK.TOKrcurly;     return;



            case '?': p++; t.value = TOK.TOKquestion;   return;



            case ',': p++; t.value = TOK.TOKcomma;              return;



            case ';': p++; t.value = TOK.TOKsemicolon;  return;



            case ':': p++; t.value = TOK.TOKcolon;              return;



            case '$': p++; t.value = TOK.TOKdollar;     return;



            // DOUBLE

            case '*': p++; if (*p == '=') {

                    p++; t.value = TOK.TOKmulass;

            }

                else {

                    t.value = TOK.TOKmul;

                } return;



            case '%': p++; if (*p == '=') {

                    p++; t.value = TOK.TOKmodass;

            }

                else {

                    t.value = TOK.TOKmod;

                } return;



            case '^': p++; if (*p == '=') {

                    p++; t.value = TOK.TOKxorass;

            }

                else {

                    t.value = TOK.TOKxor;

                } return;



// removed 148	case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return;





            case '#':

                p++;

                Pragma();

                continue;



            default:

                {

                    debug writefln("    default char");

                    ubyte c = *p;

                    if (c & 0x80) {

                        uint u = decodeUTF();

                        // Check for start of unicode identifier

                        if (isUniAlpha(u)) {

                            goto case_ident;

                        }



                        if (u == PS || u == LS) {

                            loc.linnum++;

                            p++;

                            continue;

                        }

                    }

                    if (isprint(c)) {

                        error("unsupported char '%s'", cast(char)c);

                    }

                    else {

                        error("unsupported char 0x%02x", cast(ubyte)c);

                    }

                    p++;

                    continue;

                }

            }

        }

    }







    // Parse escape sequence.

    uint escapeSequence() {

        uint c;

        int  n;

        int  ndigits;



        c = *p;

        switch (c) {

        case '\'':

        case '"':

        case '?':

        case '\\':

Lconsume:

            p++;

            break;



        case 'a':       c = 7;          goto Lconsume;



        case 'b':       c = 8;          goto Lconsume;



        case 'f':       c = 12;         goto Lconsume;



        case 'n':       c = 10;         goto Lconsume;



        case 'r':       c = 13;         goto Lconsume;



        case 't':       c = 9;          goto Lconsume;



        case 'v':       c = 11;         goto Lconsume;



        case 'u':

            ndigits = 4;

            goto Lhex;



        case 'U':

            ndigits = 8;

            goto Lhex;



        case 'x':

            ndigits = 2;

Lhex:

            p++;

            c = *p;

            if (ishex(c)) {

                uint v;

                n = 0;

                v = 0;

                while (1) {

                    if (isdigit(c)) {

                        c -= '0';

                    }

                    else if (islower(c)) {

                        c -= 'a' - 10;

                    }

                    else {

                        c -= 'A' - 10;

                    }

                    v = v * 16 + c;

                    c = *++p;

                    if (++n == ndigits) {

                        break;

                    }

                    if (!ishex(c)) {

                        error("escape hex sequence has %d hex digits instead of %d", n, ndigits);

                        break;

                    }

                }

//!				    if( ndigits != 2 && !utf_isValidDchar(v))

//!						error("invalid UTF character \\U%08x", v);

                c = v;

            }

            else {

                error("undefined escape hex sequence \\%s\n", c);

            }

            break;



        case '&':                                       // named character entity

            for (ubyte *idstart = ++p; 1; p++) {

                switch (*p) {

                case ';':

                    //!!!

                    /+

                     * c = HtmlNamedEntity(idstart, p - idstart);

                     * if( c == ~0 )

                     * {

                     *      error("unnamed character entity &%.*s;", p - idstart, idstart);

                     *      c = ' ';

                     * }

                     *

                     * p++;

                     +/

                    break;



                default:

                    if (isalpha(*p) || (p != idstart + 1 && isdigit(*p))) {

                        continue;

                    }

                    error("unterminated named entity");

                    break;

                }

                break;

            }

            break;



        case 0:

        case 0x1a:                                      // end of file

            c = '\\';

            break;



        default:

            if (isoctal(c)) {

                ubyte v;

                n = 0;

                do {

                    v = v * 8 + (c - '0');

                    c = *++p;

                } while (++n < 3 && isoctal(c));

                c = v;

            }

            else {

                error("undefined escape sequence \\%s\n", c);

            }

            break;

        }

        return(c);

    }



    /**************************************

     */



    TOK wysiwygStringConstant(Token *t, int tc) {

        uint c;

        Loc  start = loc;



        p++;

        stringbuffer.offset = 0;

        while (1) {

            c = *p++;

            switch (c) {

            case '\n':

                loc.linnum++;

                break;



            case '\r':

                if (*p == '\n') {

                    continue;                           // ignore

                }

                c = '\n';                               // treat EndOfLine as \n character

                loc.linnum++;

                break;



            case 0:

            case 0x1a:

                error("unterminated string constant starting at %s", start.toChars());

                t.ustring = "";

                t.postfix = 0;

                return(TOK.TOKstring);



            case '"':

            case '`':

                if (c == tc) {

//				    t.len = stringbuffer.offset;

                    stringbuffer.write(cast(byte)0);

                    t.ustring = stringbuffer.toString;

//				    t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);

//				    memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);

                    stringPostfix(t);

                    return(TOK.TOKstring);

                }

                break;



            default:

                if (c & 0x80) {

                    p--;

                    uint u = decodeUTF();

                    p++;

                    if (u == PS || u == LS) {

                        loc.linnum++;

                    }

                    stringbuffer.write(u);

                    continue;

                }

                break;

            }

            stringbuffer.write(c);

        }

    }



    /**************************************

     * Lex hex strings:

     *	x"0A ae 34FE BD"

     */



    TOK hexStringConstant(Token *t) {

        uint c;

        Loc  start = loc;

        uint n     = 0;

        uint v;



        p++;

        stringbuffer.offset = 0;

        while (1) {

            c = *p++;

            switch (c) {

            case ' ':

            case '\t':

            case '\v':

            case '\f':

                continue;                                               // skip white space



            case '\r':

                if (*p == '\n') {

                    continue;                                           // ignore

                }



            // Treat isolated '\r' as if it were a '\n'

            case '\n':

                loc.linnum++;

                continue;



            case 0:

            case 0x1a:

                error("unterminated string constant starting at %s", start.toChars());

                t.ustring = "";

                t.postfix = 0;

                return(TOK.TOKstring);



            case '"':

                if (n & 1) {

                    error("odd number (%d) of hex characters in hex string", n);

                    stringbuffer.write(v);

                }

//				t.len = stringbuffer.offset;

//				stringbuffer.write(cast(byte)0);

                t.ustring = stringbuffer.toString;

//				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);

//				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);

                stringPostfix(t);

                return(TOK.TOKstring);



            default:

                if (c >= '0' && c <= '9') {

                    c -= '0';

                }

                else if (c >= 'a' && c <= 'f') {

                    c -= 'a' - 10;

                }

                else if (c >= 'A' && c <= 'F') {

                    c -= 'A' - 10;

                }

                else if (c & 0x80) {

                    p--;

                    uint u = decodeUTF();

                    p++;

                    if (u == PS || u == LS) {

                        loc.linnum++;

                    }

                    else {

                        error("non-hex character \\u%x", u);

                    }

                }

                else {

                    error("non-hex character '%s'", c);

                }

                if (n & 1) {

                    v = (v << 4) | c;

                    stringbuffer.write(v);

                }

                else {

                    v = c;

                }

                n++;

                break;

            }

        }

    }



    /**************************************

     */



    TOK escapeStringConstant(Token *t, int wide) {

        uint c;

        Loc  start = loc;



        p++;

        stringbuffer.offset = 0;

        //    debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) );

        while (1) {

            c = *p++;

            switch (c) {

            case '\\':

                switch (*p) {

                case 'u':

                case 'U':

                case '&':

                    c = escapeSequence();

                    stringbuffer.write(c);

                    continue;



                default:

                    c = escapeSequence();

                    break;

                }

                break;



            case '\n':

                loc.linnum++;

                break;



            case '\r':

                if (*p == '\n') {

                    continue;                           // ignore

                }

                c = '\n';                               // treat EndOfLine as \n character

                loc.linnum++;

                break;



            case '"':

//			        writefln( "end of string: ", stringbuffer.toString );

                t.ustring = stringbuffer.toString().dup;

                //				t.len = stringbuffer.offset;

                //				stringbuffer.write(cast(byte)0);

                //				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);

                //				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);

                stringPostfix(t);



                return(TOK.TOKstring);



            case 0:

            case 0x1a:

                p--;

                error("unterminated string constant starting at %s", start.toChars());

                t.ustring = "";

//					t.len = 0;

                t.postfix = 0;

                return(TOK.TOKstring);



            default:

                if (c & 0x80) {

                    p--;

                    c = decodeUTF();

                    if (c == LS || c == PS) {

                        c = '\n';

                        loc.linnum++;

                    }

                    p++;

                    stringbuffer.write(cast(char)c);

                    continue;

                }

                break;

            }

            stringbuffer.write(cast(char)c);

//			writefln( stringbuffer.toString );

        }

    }



    //**************************************

    TOK charConstant(Token *t, int wide) {

        uint c;

        TOK  tk = TOK.TOKcharv;



        //printf("Lexer.charConstant\n");

        p++;

        c = *p++;

        switch (c) {

        case '\\':

            switch (*p) {

            case 'u':

                t.uns64value = escapeSequence();

                tk           = TOK.TOKwcharv;

                break;



            case 'U':

            case '&':

                t.uns64value = escapeSequence();

                tk           = TOK.TOKdcharv;

                break;



            default:

                t.uns64value = escapeSequence();

                break;

            }

            break;



        case '\n':

L1:

            loc.linnum++;



        case '\r':

        case 0:

        case 0x1a:

        case '\'':

            error("unterminated character constant");

            return(tk);



        default:

            if (c & 0x80) {

                p--;

                c = decodeUTF();

                p++;

                if (c == LS || c == PS) {

                    goto L1;

                }

                if (c < 0xd800 || (c >= 0xe000 && c < 0xfffe)) {

                    tk = TOK.TOKwcharv;

                }

                else {

                    tk = TOK.TOKdcharv;

                }

            }

            t.uns64value = c;

            break;

        }



        if (*p != '\'') {

            error("unterminated character constant");

            return(tk);

        }

        p++;

        return(tk);

    }



    // Get postfix of string literal.

    void stringPostfix(Token *t) {

        switch (*p) {

        case 'c':

        case 'w':

        case 'd':

            t.postfix = *p;

            p++;

            break;



        default:

            t.postfix = 0;

            break;

        }

    }



    /***************************************

     * Read \u or \U unicode sequence

     * Input:

     *	u	'u' or 'U'

     */

    /*

     * uint Wchar(uint u)

     * {

     *  uint value;

     *  uint n;

     *  ubyte c;

     *  uint nchars;

     *

     *  nchars = (u == 'U') ? 8 : 4;

     *  value = 0;

     *  for (n = 0; 1; n++)

     *  {

     * ++p;

     *              if( n == nchars)

     *                  break;

     *              c = *p;

     *              if( !ishex(c))

     *              {

     *                      error("\\%s sequence must be followed by %d hex characters", u, nchars);

     *                  break;

     *              }

     *              if( isdigit(c))

     *                  c -= '0';

     *              else if( islower(c))

     *                  c -= 'a' - 10;

     *              else

     *                  c -= 'A' - 10;

     *              value <<= 4;

     *              value |= c;

     *  }

     *  return value;

     * }

     */



    /**************************************

     * Read in a number.

     * If it's an integer, store it in tok.TKutok.Vlong.

     *	integers can be decimal, octal or hex

     *	Handle the suffixes U, UL, LU, L, etc.

     * If it's double, store it in tok.TKutok.Vdouble.

     * Returns:

     *	TKnum

     *	TKdouble,...

     */



    TOK number(Token *t) {

        //debug writefln("Lexer.number()");

        // We use a state machine to collect numbers

        enum STATE {

            STATE_initial,

            STATE_0,

            STATE_decimal,

            STATE_octal,

            STATE_octale,

            STATE_hex,

            STATE_binary,

            STATE_hex0,

            STATE_binary0,

            STATE_hexh,

            STATE_error

        }



        enum FLAGS {

            FLAGS_decimal  = 1,                         // decimal

            FLAGS_unsigned = 2,                         // u or U suffix

            FLAGS_long     = 4,                         // l or L suffix

        }

        FLAGS flags = FLAGS.FLAGS_decimal;



        int   i;

        TOK   result;

        int   base;



        stringbuffer.offset = 0;

//		stringbuffer.data = null;

        STATE state   = STATE.STATE_initial;

        ubyte * start = p;



        TOK _isreal() {

            p = start;

            return(inreal(t));

        }



        while (true) {

            char c = cast(char)*p;

            switch (state) {

            case STATE.STATE_initial:                           // opening state

                if (c == '0') {

                    state = STATE.STATE_0;

                }

                else {

                    state = STATE.STATE_decimal;

                }

                break;



            case STATE.STATE_0:

                flags = cast(FLAGS)(flags & ~FLAGS.FLAGS_decimal);

                switch (c) {

                //	#if ZEROH

//					    case 'H':			// 0h

//					    case 'h':

//							goto hexh;

                //	#endif

                case 'X':

                case 'x':

                    state = STATE.STATE_hex0;

                    break;



                case '.':

                    if (p[1] == '.') {                                          // .. is a separate token

                        goto done;

                    }



                case 'i':

                case 'f':

                case 'F':

                    goto _Real;



                //	#if ZEROH

//					    case 'E':

//					    case 'e':

//							goto case_hex;

                //	#endif

                case 'B':

                case 'b':

                    state = STATE.STATE_binary0;

                    break;



                case '0':

                case '1':

                case '2':

                case '3':

                case '4':

                case '5':

                case '6':

                case '7':

                    state = STATE.STATE_octal;

                    break;



                //	#if ZEROH

//					    case '8': case '9': case 'A':

//					    case 'C': case 'D': case 'F':

//					    case 'a': case 'c': case 'd': case 'f':

//						    case_hex:

//							state = STATE.STATE_hexh;

//							break;

                //	#endif

                case '_':

                    state = STATE.STATE_octal;

                    p++;

                    continue;



                default:

                    goto done;

                }

                break;



            case STATE.STATE_decimal:                           // reading decimal number



                // if its not a digit - decimal complete or not a decimal

                if (!isdigit(c)) {

//						debug writefln( "\tnon-digit( %s )", c );

                    //	#if ZEROH

//					    if( ishex(c) || c == 'H' || c == 'h' )

//							goto hexh;

                    //	#endif

                    //! wtf ?

                    // ignore embedded _

                    if (c == '_') {

                        p++;

                        continue;

                    }



                    // check decimal point - make real

                    if (c == '.' && p[1] != '.') {

                        goto _Real;

                    }



                    // check for mantra - make real

                    if (c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E') {

_Real:                  // It's a real number. Back up and rescan as a real

                        p = start;

                        return(inreal(t));

                    }



                    goto done;

                }

                break;



            case STATE.STATE_hex0:                              // reading hex number

            case STATE.STATE_hex:

                if (!ishex(c)) {

                    if (c == '_') {                                     // ignore embedded _

                        p++;

                        continue;

                    }

                    if (c == '.' && p[1] != '.') {

                        goto _Real;

                    }

                    if (c == 'P' || c == 'p' || c == 'i') {

                        goto _Real;

                    }

                    if (state == STATE.STATE_hex0) {

                        error("Hex digit expected, not '%s'", c);

                    }

                    goto done;

                }

                state = STATE.STATE_hex;

                break;



            //	#if ZEROH

//				    hexh:

//						state = STATE.STATE_hexh;

//

//				    case STATE.STATE_hexh:		// parse numbers like 0FFh

//						if( !ishex(c))

//						{

//						    if( c == 'H' || c == 'h')

//						    {

//								p++;

//								base = 16;

//								goto done;

//						    }

//						    else

//						    {

//								// Check for something like 1E3 or 0E24

//								if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset))

//								    goto _Real;

//								error("Hex digit expected, not '%s'", c);

//								goto done;

//						    }

//						}

//						break;

            //		#endif



            case STATE.STATE_octal:                                     // reading octal number

            case STATE.STATE_octale:                                    // reading octal number with non-octal digits

                if (!isoctal(c)) {

//				#if ZEROH

//						    if( ishex(c) || c == 'H' || c == 'h' )

//								goto hexh;

//				#endif

                    if (c == '_') {                                             // ignore embedded _

                        p++;

                        continue;

                    }

                    if (c == '.' && p[1] != '.') {

                        goto _Real;

                    }

                    if (c == 'i') {

                        goto _Real;

                    }

                    if (isdigit(c)) {

                        state = STATE.STATE_octale;

                    }

                    else {

                        goto done;

                    }

                }

                break;



            case STATE.STATE_binary0:                                   // starting binary number

            case STATE.STATE_binary:                                    // reading binary number

                if (c != '0' && c != '1') {

                    //			#if ZEROH

//						    if( ishex(c) || c == 'H' || c == 'h' )

//								goto hexh;

                    //			#endif

                    if (c == '_') {                                             // ignore embedded _

                        p++;

                        continue;

                    }

                    if (state == STATE.STATE_binary0) {

                        error("binary digit expected");

                        state = STATE.STATE_error;

                        break;

                    }

                    else {

                        goto done;

                    }

                }

                state = STATE.STATE_binary;

                break;



            case STATE.STATE_error:                                     // for error recovery

                if (!isdigit(c)) {                                      // scan until non-digit

                    goto done;

                }

                break;



            default:

                assert(0);

            }

            stringbuffer.write(cast(char)c);

            p++;

        }

done:

        stringbuffer.write(cast(char)0);                // terminate string



//		debug writefln( "\tdigit complete( %s )", stringbuffer.toString );



        if (state == STATE.STATE_octale) {

            error("Octal digit expected");

        }



        uinteger_t n;                           // unsigned >=64 bit integer type



        if (stringbuffer.offset == 2 && (state == STATE.STATE_decimal || state == STATE.STATE_0)) {

            n = stringbuffer.data[0] - '0';

        }

        else {

            // Convert string to integer

            char* p = cast(char*)stringbuffer.data.ptr;

            int r   = 10;

            int d;

            if (*p == '0') {

                if (p[1] == 'x' || p[1] == 'X') {

                    // "0x#"

                    p += 2;

                    r  = 16;

                }

                else if (p[1] == 'b' || p[1] == 'B') {

                    // "0b#" - binary

                    p += 2;

                    r  = 2;

                }

                else if (isdigit(p[1])) {

                    p += 1;

                    r  = 8;

                }

            }



            n = 0;



            while (true) {

                if (*p >= '0' && *p <= '9') {

                    d = *p - '0';

                }

                else if (*p >= 'a' && *p <= 'z') {

                    d = *p - 'a' + 10;

                }

                else if (*p >= 'A' && *p <= 'Z') {

                    d = *p - 'A' + 10;

                }

                else {

                    break;

                }



                if (d >= r) {

                    break;

                }



                if (n * r + d < n) {

                    error("integer overflow");

                    break;

                }



                n = n * r + d;

                p++;

            }



            // if n needs more than 64 bits

            if (n.sizeof > 8 && n > 0xffffffffffffffffL) {

                error("integer overflow");

            }

        }



        // Parse trailing 'u', 'U', 'l' or 'L' in any combination

        while (true) {

            ubyte f;

            switch (*p) {

            case 'U':

            case 'u':

                f = FLAGS.FLAGS_unsigned;

                goto L1;



            case 'L':

            case 'l':

                f = FLAGS.FLAGS_long;

L1:

                p++;

                if (flags & f) {

                    error("unrecognized token");

                }

                flags = cast(FLAGS)(flags | f);

                continue;



            default:

                break;

            }

            break;

        }



        switch (flags) {

        case 0:

            /* Octal or Hexadecimal constant.

             * First that fits: int, uint, long, ulong

             */

            if (n & 0x8000000000000000L) {

                result = TOK.TOKuns64v;

            }

            else if (n & 0xffffffff00000000L) {

                result = TOK.TOKint64v;

            }

            else if (n & 0x80000000) {

                result = TOK.TOKuns32v;

            }

            else {

                result = TOK.TOKint32v;

            }

            break;



        case FLAGS.FLAGS_decimal:

            /* First that fits: int, long, long long

             */

            if (n & 0x8000000000000000L) {

                error("signed integer overflow");

                result = TOK.TOKuns64v;

            }

            else if (n & 0xffffffff80000000L) {

                result = TOK.TOKint64v;

            }

            else {

                result = TOK.TOKint32v;

            }

            break;



        case FLAGS.FLAGS_unsigned:

        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned:

            /* First that fits: uint, ulong

             */

            if (n & 0xffffffff00000000L) {

                result = TOK.TOKuns64v;

            }

            else {

                result = TOK.TOKuns32v;

            }

            break;



        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long:

            if (n & 0x8000000000000000L) {

                error("signed integer overflow");

                result = TOK.TOKuns64v;

            }

            else {

                result = TOK.TOKint64v;

            }

            break;



        case FLAGS.FLAGS_long:

            if (n & 0x8000000000000000L) {

                result = TOK.TOKuns64v;

            }

            else {

                result = TOK.TOKint64v;

            }

            break;



        case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:

        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:

            result = TOK.TOKuns64v;

            break;



        default:

            debug writefln("%x", flags);

            assert(0);

        }

        t.uns64value = n;

        return(result);

    }



    /**************************************

     * Read in characters, converting them to real.

     * Bugs:

     *	Exponent overflow not detected.

     *	Too much requested precision is not detected.

     */



    TOK inreal(Token *t) {

        int  dblstate;

        uint c;

        char hex;                       // is this a hexadecimal-floating-constant?

        TOK  result;



        //printf("Lexer.inreal()\n");

        stringbuffer.offset = 0;

        dblstate            = 0;

        hex                 = 0;

Lnext:

        while (1) {

            // Get next char from input

            c = *p++;

            //printf("dblstate = %d, c = '%s'\n", dblstate, c);

            while (1) {

                switch (dblstate) {

                case 0:                                 // opening state

                    if (c == '0') {

                        dblstate = 9;

                    }

                    else if (c == '.') {

                        dblstate = 3;

                    }

                    else {

                        dblstate = 1;

                    }

                    break;



                case 9:

                    dblstate = 1;

                    if (c == 'X' || c == 'x') {

                        hex++;

                        break;

                    }



                case 1:                                 // digits to left of .

                case 3:                                 // digits to right of .

                case 7:                                 // continuing exponent digits

                    if (!isdigit(c) && !(hex && isxdigit(c))) {

                        if (c == '_') {

                            goto Lnext;                         // ignore embedded '_'

                        }

                        dblstate++;

                        continue;

                    }

                    break;



                case 2:                                 // no more digits to left of .

                    if (c == '.') {

                        dblstate++;

                        break;

                    }



                case 4:                                 // no more digits to right of .

                    if ((c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p')) {

                        dblstate = 5;

                        hex      = 0;                           // exponent is always decimal

                        break;

                    }

                    if (hex) {

                        error("binary-exponent-part required");

                    }

                    goto done;



                case 5:                                 // looking immediately to right of E

                    dblstate++;

                    if (c == '-' || c == '+') {

                        break;

                    }



                case 6:                                 // 1st exponent digit expected

                    if (!isdigit(c)) {

                        error("exponent expected");

                    }

                    dblstate++;

                    break;



                case 8:                                 // past end of exponent digits

                    goto done;

                }

                break;

            }

            stringbuffer.write(c);

        }

done:

        p--;



        stringbuffer.write(cast(byte)0);



//	#if _WIN32 && __DMC__

        char *save = __locale_decpoint;

        __locale_decpoint = ".";

//	#endif

        t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null);

        errno          = 0;

        switch (*p) {

        case 'F':

        case 'f':

            strtof(cast(char *)stringbuffer.data.ptr, null);

            result = TOK.TOKfloat32v;

            p++;

            break;



        default:

            strtod(cast(char *)stringbuffer.data.ptr, null);

            result = TOK.TOKfloat64v;

            break;



        case 'L':

        case 'l':

            result = TOK.TOKfloat80v;

            p++;

            break;

        }

        if (*p == 'i' || *p == 'I') {

            p++;

            switch (result) {

            case TOK.TOKfloat32v:

                result = TOK.TOKimaginary32v;

                break;



            case TOK.TOKfloat64v:

                result = TOK.TOKimaginary64v;

                break;



            case TOK.TOKfloat80v:

                result = TOK.TOKimaginary80v;

                break;

            }

        }

//	#if _WIN32 && __DMC__

        __locale_decpoint = save;

//	#endif

        if (errno == ERANGE) {

            error("number is not representable");

        }

        return(result);

    }









    /*********************************************

     * Do pragma.

     * Currently, the only pragma supported is:

     *	#line linnum [filespec]

     */



    void Pragma() {

        Token  tok;

        int    linnum;



        char[] filespec;

        Loc    loc = this.loc;



        scan(&tok);



        if (tok.value != TOK.TOKidentifier || tok.ident != Id.line) {

            goto Lerr;

        }



        scan(&tok);

        if (tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v) {

            linnum = tok.uns64value - 1;

        }

        else {

            goto Lerr;

        }



        while (1) {

            switch (*p) {

            case 0:

            case 0x1a:

            case '\n':

Lnewline:

                this.loc.linnum = linnum;

                if (filespec.length) {

                    this.loc.filename = filespec;

                }

                return;



            case '\r':

                p++;

                if (*p != '\n') {

                    p--;

                    goto Lnewline;

                }

                continue;



            case ' ':

            case '\t':

            case '\v':

            case '\f':

                p++;

                continue;                                               // skip white space



            case '_':

                if (mod && memcmp(p, cast(char*)"__FILE__", 8) == 0) {

                    p += 8;

//!					    filespec = mem.strdup(loc.filename ? loc.filename : mod.ident.toChars());

                }

                continue;



            case '"':

                if (filespec) {

                    goto Lerr;

                }

                stringbuffer.offset = 0;

                p++;

                while (1) {

                    uint c;

                    c = *p;

                    switch (c) {

                    case '\n':

                    case '\r':

                    case 0:

                    case 0x1a:

                        goto Lerr;



                    case '"':

                        stringbuffer.write(cast(byte)0);

                        //						    filespec = mem.strdup((char *)stringbuffer.data);

                        filespec = stringbuffer.toString.dup;

                        p++;

                        break;



                    default:

                        if (c & 0x80) {

                            uint u = decodeUTF();

                            if (u == PS || u == LS) {

                                goto Lerr;

                            }

                        }

                        stringbuffer.write(c);

                        p++;

                        continue;

                    }

                    break;

                }

                continue;



            default:

                if (*p & 0x80) {

                    uint u = decodeUTF();

                    if (u == PS || u == LS) {

                        goto Lnewline;

                    }

                }

                goto Lerr;

            }

        }



Lerr:

        errorLoc(loc, "#line integer [\"filespec\"]\\n expected");

    }







    /***************************************************

     * Parse doc comment embedded between t.ptr and p.

     * Remove trailing blanks and tabs from lines.

     * Replace all newlines with \n.

     * Remove leading comment character from each line.

     * Decide if it's a lineComment or a blockComment.

     * Append to previous one for this token.

     */



    void getDocComment(Token *t, uint lineComment) {

        auto OutBuffer buf       = new OutBuffer;

        ubyte          ct        = t.ptr[2];

        ubyte          *q        = t.ptr + 3; // start of comment text

        int            linestart = 0;



        ubyte          *qend = p;



        if (ct == '*' || ct == '+') {

            qend -= 2;

        }



        // Scan over initial row of ****'s or ++++'s or ////'s

        for (; q < qend; q++) {

            if (*q != ct) {

                break;

            }

        }



        // Remove trailing row of ****'s or ++++'s

        if (ct != '/') {

            for (; q < qend; qend--) {

                if (qend[-1] != ct) {

                    break;

                }

            }

        }



        for (; q < qend; q++) {

            ubyte c = *q;



            switch (c) {

            case '*':

            case '+':

                if (linestart && c == ct) {

                    linestart = 0;

                    // Trim preceding whitespace up to preceding \n

                    while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) {

                        buf.offset--;

                    }

                    continue;

                }

                break;



            case ' ':

            case '\t':

                break;



            case '\r':

                if (q[1] == '\n') {

                    continue;                                   // skip the \r

                }

                goto Lnewline;



            default:

                if (c == 226) {

                    // If LS or PS

                    if (q[1] == 128 &&

                        (q[2] == 168 || q[2] == 169)) {

                        q += 2;

                        goto Lnewline;

                    }

                }

                linestart = 0;

                break;



Lnewline:

                c = '\n';                               // replace all newlines with \n



            case '\n':

                linestart = 1;



                // Trim trailing whitespace

                while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) {

                    buf.offset--;

                }



                break;

            }

            buf.write(c);

        }



        // Always end with a newline

        if (!buf.offset || buf.data[buf.offset - 1] != '\n') {

            buf.writenl();

        }



        //buf.write(cast(char)0);



        // It's a line comment if the start of the doc comment comes

        // after other non-whitespace on the same line.

//	    ubyte** dc = (lineComment && anyToken)

//				 ? &t.lineComment

//				 : &t.blockComment;



        char[] dc = (lineComment && anyToken) ? t.lineComment : t.blockComment;



        // Combine with previous doc comment, if any

        if (dc.length) {

            dc = combineComments(dc, buf.toString().dup);

        }

        else {

            dc = buf.toString().dup;

        }



//		writefln( dc );



        if (lineComment && anyToken) {

            t.lineComment = dc;

        }

        else {

            t.blockComment = dc;

        }

    }

}



// character maps

static ubyte[256] cmtable;



const int CMoctal  = 0x1;

const int CMhex    = 0x2;

const int CMidchar = 0x4;



ubyte isoctal(ubyte c) {

    return(cmtable[c] & CMoctal);

}

ubyte ishex(ubyte c) {

    return(cmtable[c] & CMhex);

}

ubyte isidchar(ubyte c) {

    return(cmtable[c] & CMidchar);

}



static void cmtable_init() {

    for (uint c = 0; c < cmtable.length; c++) {

        if ('0' <= c && c <= '7') {

            cmtable[c] |= CMoctal;

        }

        if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) {

            cmtable[c] |= CMhex;

        }

        if (isalnum(c) || c == '_') {

            cmtable[c] |= CMidchar;

        }

    }

}





/+

 * struct StringValue

 * {

 *  union

 *  {

 *              int intvalue;

 *              void *ptrvalue;

 *              dchar *string;

 *  }

 *

 *  char[] lstring;

 * }

 * #define CASE_BASIC_TYPES

 *      case TOKwchar: case TOKdchar:

 *      case TOKbit: case TOKbool: case TOKchar:

 *      case TOKint8: case TOKuns8:

 *      case TOKint16: case TOKuns16:

 *      case TOKint32: case TOKuns32:

 *      case TOKint64: case TOKuns64:

 *      case TOKfloat32: case TOKfloat64: case TOKfloat80:

 *      case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:

 *      case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:

 *      case TOKvoid:

 *

 * #define CASE_BASIC_TYPES_X(t)					\

 *      case TOKvoid:	 t = Type::tvoid;  goto LabelX;		\

 *      case TOKint8:	 t = Type::tint8;  goto LabelX;		\

 *      case TOKuns8:	 t = Type::tuns8;  goto LabelX;		\

 *      case TOKint16:	 t = Type::tint16; goto LabelX;		\

 *      case TOKuns16:	 t = Type::tuns16; goto LabelX;		\

 *      case TOKint32:	 t = Type::tint32; goto LabelX;		\

 *      case TOKuns32:	 t = Type::tuns32; goto LabelX;		\

 *      case TOKint64:	 t = Type::tint64; goto LabelX;		\

 *      case TOKuns64:	 t = Type::tuns64; goto LabelX;		\

 *      case TOKfloat32: t = Type::tfloat32; goto LabelX;	\

 *      case TOKfloat64: t = Type::tfloat64; goto LabelX;	\

 *      case TOKfloat80: t = Type::tfloat80; goto LabelX;	\

 *      case TOKimaginary32: t = Type::timaginary32; goto LabelX;	\

 *      case TOKimaginary64: t = Type::timaginary64; goto LabelX;	\

 *      case TOKimaginary80: t = Type::timaginary80; goto LabelX;	\

 *      case TOKcomplex32: t = Type::tcomplex32; goto LabelX;	\

 *      case TOKcomplex64: t = Type::tcomplex64; goto LabelX;	\

 *      case TOKcomplex80: t = Type::tcomplex80; goto LabelX;	\

 *      case TOKbit:	 t = Type::tbit;     goto LabelX;	\

 *      case TOKchar:	 t = Type::tchar;    goto LabelX;	\

 *      case TOKwchar:	 t = Type::twchar; goto LabelX;	\

 *      case TOKdchar:	 t = Type::tdchar; goto LabelX;	\

 *      LabelX

 +/