/tests/input/d/Lexer.d

/+

	Copyright (c) 1999-2006 by Digital Mars

	All Rights Reserved

	written by Walter Bright www.digitalmars.com	

	License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt.

	See the included readme.txt for details.

	D Language conversion by: J Duncan

+/



/**

	d language lexer 

*/



module dparser.Lexer;



import dparser.Root;



import dparser.Tokens;

import dparser.Token;

import dparser.Keyword;



import dparser.Types;



import dparser.Module;

import dparser.Identifier;

import dparser.unialpha;



import dparser.OutBuffer;



//private import std.ctype;

//private import std.string;

//import dwf.core.debugapi;



int errno = 0;



//#if _WIN32 && __DMC__

// from \dm\src\include\setlocal.h

//extern "C" char * __cdecl __locale_decpoint;

char* __locale_decpoint;

//#endif

//const uint LS = 0x2028;	// UTF line separator

//const uint PS = 0x2029;	// UTF paragraph separator



//extern int isUniAlpha(unsigned u);

//extern int HtmlNamedEntity(unsigned char *p, int length);



/**

	Lexer object

*/



class Lexer

{

   

	static Identifier[char[]] 	stringtable;

    static OutBuffer 		stringbuffer;

    static Token*			freelist;



    Token token;		// current token

    Module mod;			// current module

    Loc loc;			// for error messages

    ubyte *base;		// pointer to start of buffer

    ubyte *end;			// past end of buffer

    ubyte *p;			// current character

    int doDocComment;	// collect doc comment information

    int anyToken;		// !=0 means seen at least one token

    int commentToken;	// !=0 means comments are TOKcomment's





    this( Module mod, ubyte* base, uint begoffset, uint endoffset, int doDocComment, int commentToken )

	{

	    if( stringbuffer is null )

		    stringbuffer = new OutBuffer;

	    loc = Loc(mod, 1);



	    this.base 			= base;

	    this.end  			= base + endoffset;

	    this.p 				= base + begoffset;

	    this.mod 			= mod;

	    this.doDocComment	= doDocComment;

	    this.commentToken	= commentToken;



		/* 

			If first line starts with '#!', ignore the line

		*/



		if( p[0] == '#' && p[1] =='!' )

		{

			p += 2;

			while( true )

			{   

				ubyte c = *p;

				switch( c )

				{

					case '\n':

						p++;

						break;



					case '\r':

						p++;

						if( *p == '\n' )

							p++;

						break;



					case 0:

					case 0x1A:

						break;



					default:

						if( c & 0x80 )

						{   

							uint u = decodeUTF();

							if( u == PS || u == LS )

								break;

						}

						p++;

						continue;

				}

				break;

			}



			loc.linnum = 2;

		}



	}







	// generate a unique identifier for this string

    static Identifier idPool( in char[] str )

    {

//	    StringValue sv;

//	    uint len = s.length;

//	    StringValue sv = stringtable.update(s, len);

//	    Identifier* id = cast(Identifier*) sv.ptrvalue;

//	    if( id is null )

	    if( (str in stringtable) == null )

	    {

			stringtable[str] = new Identifier( str, TOK.TOKidentifier );

	    }

	    return stringtable[str];

	}



    static void initKeywords()

	{

		// build character map

	    cmtable_init();

		

		// create keyword tokens & identifiers

		dparser.Keyword.initKeywords();



		// create standard lexer tokens

		dparser.Token.createLexerTokens();

	}



	// Combine two document comments into one.

    static char[] combineComments( char[] c1, char[] c2 )

	{

	    char[] c = c2;

	    if( c1.length )

	    {

			c = c1;

			if( c2.length )

			{

				c = c1 ~ "\n" ~ c2;

			}

	    }

	    return c;

	}



	// Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence.

	//! fix

	uint decodeUTF()

	{

	    ubyte* s = p;

	    ubyte  c = *s;



	    assert( c & 0x80 );

		if( !(c & 0x80) )

			return c;



		return cast(uint) 'X';

	/*

	    dchar u;

	    uint len;







	    // Check length of remaining string up to 6 UTF-8 characters

	    for( len = 1; len < 6 && s[len]; len++ )

	    {



	    }

			/+

	    uint idx = 0;

	    char* msg = utf_decodeChar( s, len, &idx, &u );

	    p += idx - 1;

	    if( msg )

	    {

			error(msg);

	    }

			+/

	    return u;

	*/

	}



	void error( ... )

	{

	    if( (mod !is null) && !global.gag )

	    {

			writefln( formatLoc( loc, _arguments, _argptr ) );

			/*

			char[] p = loc.toChars();

			if( p.length )

			    writef( "%s: ", p );

		    writefx( stdout, _arguments, _argptr, 1 );

			*/

			if( global.errors >= global.max_errors )	// moderate blizzard of cascading messages

		    	throw new Exception( "too many errors" );

	    }



	    global.errors++;

	}



	void errorLoc(Loc loc, ...)

	{

	    if( (mod !is null) && !global.gag )

	    {

			writefln( formatLoc( loc, _arguments, _argptr ) );

			/*

			char[] p = loc.toChars();

			if( p.length )

			    writef("%s: ", p);

		    writefx(stdout, _arguments, _argptr, 1);

			*/

			if( global.errors >= 20 )	// moderate blizzard of cascading messages

		    	throw new Exception( "too many errors" );

	    }



	    global.errors++;

	}





	TOK nextToken()

	{

	    if( token.next )

	    {

			Token* t = token.next;

			memcpy( &token, t, Token.sizeof );

//			t.next = freelist;

//			freelist = t;

	    }

	    else

	    {

			scan( &token );

	    }

//	    token.print();

	    return token.value;

	}



	Token* peek( inout Token ct )

	{

		Token* t;

	    if( ct.next )

			t = ct.next;

	    else

	    {

			t = new Token;

			scan( t );

			t.next = null;

			ct.next = t;

	    }

	    return t;

	}



	// Turn next token in buffer into a token.



	void scan( Token* t )

	{

//		debug writefln("scan token");

	    uint lastLine = loc.linnum;

	    uint linnum;

	    t.blockComment 	= null;

	    t.lineComment 	= null;

	    while( true )

	    {

			t.ptr = p;

//			debug writefln( "    p = %d, *p = ", cast(uint)p, cast(char)*p );

			switch( *p )

			{

			    case 0:

			    case 0x1a:

					t.value = TOK.TOKeof;			// end of file

//					debug writefln( "    EOF" );

					return;



			    case ' ':

			    case '\t':

			    case '\v':

			    case '\f':

					p++;

//					debug writefln( "    whitespace" );

					continue;					// skip white space



			    case '\r':

//					debug writefln( "    cr" );

					p++;

					if( *p != '\n' )			// if CR stands by itself

					    loc.linnum++;

					continue;					// skip white space



			    case '\n':

//					debug writefln( "    nl" );

					p++;

					loc.linnum++;

					continue;					// skip white space



			    case '0':  	case '1':   case '2':   case '3':   case '4':

			    case '5':  	case '6':   case '7':   case '8':   case '9':

					t.value = number(t);

					return;



/* 

	#if CSTRINGS

			    case '\'':

				t.value = charConstant(t, 0);

				return;



			    case '"':

				t.value = stringConstant(t,0);

				return;



			    case 'l':

			    case 'L':

				if( p[1] == '\'')

				{

				    p++;

				    t.value = charConstant(t, 1);

				    return;

				}

				else if( p[1] == '"')

				{

				    p++;

				    t.value = stringConstant(t, 1);

				    return;

				}

	#else 

*/

			    case '\'':

//					debug writefln( "    char" );

					t.value = charConstant(t,0);

					return;



			    case 'r':

//					debug writefln( "    wysiwyg" );

					if( p[1] != '"')

					    goto case_ident;

					p++;

				    case '`':

					t.value = wysiwygStringConstant(t, *p);

					return;



			    case 'x':

//					debug writefln( "    hex string" );

					if( p[1] != '"')

					    goto case_ident;

					p++;

					t.value = hexStringConstant(t);

					return;





			    case '"':

//					debug writefln( "    string" );

					t.value = escapeStringConstant( t, 0 );

//					debug writefln( t.ustring );

					return;



			    case '\\':			// escaped string literal

//					debug writefln( "    escaped string literal" );

					uint c;

					stringbuffer.offset = 0;

					do

					{

					    p++;

					    c = escapeSequence();

					    stringbuffer.write(c);

					} while (*p == '\\');

//					t.len = stringbuffer.offset;

//					stringbuffer.write(cast(byte)0);

					t.ustring = stringbuffer.toString;

//					memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset );

					t.postfix = 0;

					t.value = TOK.TOKstring;

					return;



			    case 'l':

			    case 'L':

//	#endif



			    case 'a':  	case 'b':   case 'c':   case 'd':   case 'e':

			    case 'f':  	case 'g':   case 'h':   case 'i':   case 'j':

			    case 'k':  	            case 'm':   case 'n':   case 'o':

			    case 'p':  	case 'q': /*case 'r':*/ case 's':   case 't':

			    case 'u':  	case 'v':   case 'w': /*case 'x':*/ case 'y':

			    case 'z':

			    case 'A':  	case 'B':   case 'C':   case 'D':   case 'E':

			    case 'F':  	case 'G':   case 'H':   case 'I':   case 'J':

			    case 'K':  	            case 'M':   case 'N':   case 'O':

			    case 'P':  	case 'Q':   case 'R':   case 'S':   case 'T':

			    case 'U':  	case 'V':   case 'W':   case 'X':   case 'Y':

			    case 'Z':

			    case '_':

			    case_ident:

			    {

//					debug writefln( "    identifier" );

					ubyte c;

					do

					{

					    c = *++p;

					} while( isidchar(c) || (c & 0x80 && isUniAlpha( decodeUTF())) );



//					sv = stringtable.update((char *)t.ptr, p - t.ptr);

					char[] tmp;

					tmp.length = p - t.ptr;

					memcpy( tmp.ptr, t.ptr, p - t.ptr );

					Identifier id;

					Identifier* pid = tmp in stringtable;

					if( pid )

					{

						id = *pid;

					}



					if( id is null )

					{

						id = new Identifier( tmp, TOK.TOKidentifier );

						stringtable[tmp] = id;

					}



					t.ident = id;

					t.value = cast(TOK) id.value;

					anyToken = 1;



					// if special identifier token

					if( *t.ptr == '_')

					{

					    static char date[11+1];

					    static char time[8+1];

					    static char timestamp[24+1];



					    if( !date[0] )	// lazy evaluation

					    {

					    	//!!

					    	/+

							time_t t;

							char *p;

							.time(&t);

							p = ctime(&t);

							assert(p);

							sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20);

							sprintf(time.ptr, "%.8s", p + 11);

							sprintf(timestamp.ptr, "%.24s", p);

							+/

					    }



					    if( mod && id is Id.FILE )

					    {

							t.value = TOK.TOKstring;

							if( loc.filename.length )

								t.ustring = loc.filename;

							else

								t.ustring = mod.ident.toChars();

							goto Llen;

					    }

					    else if( mod && id == Id.LINE )

					    {

							t.value = TOK.TOKint64v;

							t.uns64value = loc.linnum;

					    }

					    else if( id == Id.DATE )

					    {

							t.value = TOK.TOKstring;

							//! t.ustring = date;

							goto Llen;

					    }

					    else if( id == Id.TIME )

					    {

							t.value = TOK.TOKstring;

							//! t.ustring = time;

							goto Llen;

					    }

					    else if( id == Id.TIMESTAMP )

					    {

							t.value = TOK.TOKstring;

							//! t.ustring = timestamp;

					     Llen:

							t.postfix = 0;

//							t.len = strlen((char *)t.ustring);

					    }

					}

					//printf("t.value = %d\n",t.value);

					return;

			    }



				// comments

			    case '/':

					p++;

					switch( *p )

					{

					    case '=':

							p++;

							t.value = TOK.TOKdivass;

							return;



					    case '*':	// '/*'

							p++;

							linnum = loc.linnum;

							while( true )

							{

							    while( true )

							    {

									ubyte c = *p;

									switch( c )

									{

									    case '/':

											break;



									    case '\n':

											loc.linnum++;

											p++;

											continue;



									    case '\r':

											p++;

											if( *p != '\n')

											    loc.linnum++;

											continue;



									    case 0:

									    case 0x1A:

											error("unterminated /* */ comment");

											p = end;

											t.value = TOK.TOKeof;

											return;



									    default:

											if( c & 0x80)

											{   

												uint u = decodeUTF();

											    if( u == PS || u == LS )

													loc.linnum++;

											}

											p++;

											continue;

									}

									break;

							    }

							    p++;

							    if( p[-2] == '*' && p - 3 != t.ptr )

									break;

							}

							

							if( commentToken )

							{

							    t.value = TOK.TOKcomment;

							    return;

							}

							// if /** but not /**/

							else  if( doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr )

							    getDocComment( t, lastLine == linnum );	//! ?

							continue;



					    case '/':		// do // style comments

							linnum = loc.linnum;

							while (1)

							{  

								ubyte c = *++p;

							    switch (c)

							    {

									case '\n':

									    break;

					

									case '\r':

									    if( p[1] == '\n')

										p++;

									    break;

					

									case 0:

									case 0x1a:

									    if( commentToken )

									    {

											p = end;

											t.value = TOK.TOKcomment;

											return;

									    }

									    if( doDocComment && t.ptr[2] == '/' )

											getDocComment( t, lastLine == linnum );

									    p = end;

									    t.value = TOK.TOKeof;

									    return;

					

									default:

									    if( c & 0x80)

									    {

											uint u = decodeUTF();

											if( u == PS || u == LS)

											    break;

									    }

									    continue;

							    }

							    break;

							}



							if( commentToken )

							{

							    p++;

							    loc.linnum++;

							    t.value = TOK.TOKcomment;

							    return;

							}

							if( doDocComment && t.ptr[2] == '/' )

							    getDocComment( t, lastLine == linnum );

				

							p++;

							loc.linnum++;

							continue;

	

					    case '+':

						    {	int nest;

								linnum = loc.linnum;

								p++;

								nest = 1;

								while (1)

								{

									ubyte c = *p;

								    switch (c)

								    {

										case '/':

										    p++;

										    if( *p == '+')

										    {

												p++;

												nest++;

										    }

										    continue;

						

										case '+':

										    p++;

										    if( *p == '/')

										    {

												p++;

												if( --nest == 0)

												    break;

										    }

										    continue;

						

										case '\r':

										    p++;

										    if( *p != '\n')

											loc.linnum++;

										    continue;

						

										case '\n':

										    loc.linnum++;

										    p++;

										    continue;

						

										case 0:

										case 0x1A:

										    error("unterminated /+ +/ comment");

										    p = end;

										    t.value = TOK.TOKeof;

										    return;

						

										default:

										    if( c & 0x80 )

										    {

												uint u = decodeUTF();

												if( u == PS || u == LS)

												    loc.linnum++;

										    }

										    p++;

										    continue;

								    }

								    break;

								}

								if( commentToken )

								{

								    t.value = TOK.TOKcomment;

								    return;

								}

								if( doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr )

								{  

									// if /++ but not /++/

								    getDocComment(t, lastLine == linnum);

								}

								continue;

					    	}

					    default:

						    break;

					}

					t.value = TOK.TOKdiv;

					return;



			    case '.':

					p++;

					if( isdigit(*p))

					{

					    p--;

					    t.value = inreal(t);

					}

					else if( p[0] == '.')

					{

					    if( p[1] == '.')

					    {

							p += 2;

							t.value = TOK.TOKdotdotdot;

					    }

					    else

					    {

							p++;

							t.value = TOK.TOKslice;

					    }

					}

					else

					    t.value = TOK.TOKdot;

					return;



			    case '&':

					p++;

					if( *p == '=')

					{   

						p++;

					    t.value = TOK.TOKandass;

					}

					else if( *p == '&')

					{   

						p++;

					    t.value = TOK.TOKandand;

					}

					else

					    t.value = TOK.TOKand;

					return;

	

				// |, ||, |=

			    case '|':

					p++;

					if( *p == '=' )

					{   p++;

					    t.value = TOK.TOKorass;

					}

					else if( *p == '|')

					{   p++;

					    t.value = TOK.TOKoror;

					}

					else

					    t.value = TOK.TOKor;

					return;

			

				    case '-':

					p++;

					if( *p == '=')

					{   p++;

					    t.value = TOK.TOKminass;

					}

					else if( *p == '-')

					{   p++;

					    t.value = TOK.TOKminusminus;

					}

					else

					    t.value = TOK.TOKmin;

					return;



				// +, +=, ++

			    case '+':

					p++;

					if( *p == '=')

					{   

						p++;

					    t.value = TOK.TOKaddass;		// +=

					}

					else if( *p == '+')

					{   p++;

					    t.value = TOK.TOKplusplus;			// ++

					}

					else

					    t.value = TOK.TOKadd;				// +

					return;



				// <, <=, <<=, <<, <>=, <>

			    case '<':

					p++;

					if( *p == '=')

					{   p++;

					    t.value = TOK.TOKle;			// <=

					}

					else if( *p == '<')

					{   p++;

					    if( *p == '=')

					    {   p++;

						t.value = TOK.TOKshlass;		// <<=

					    }

					    else

						t.value = TOK.TOKshl;			// <<

					}

					else if( *p == '>')

					{   p++;

					    if( *p == '=')

					    {   p++;

						t.value = TOK.TOKleg;			// <>=

					    }

					    else

						t.value = TOK.TOKlg;			// <>

					}

					else

					    t.value = TOK.TOKlt;			// <

					return;

		

				// >, >>, >>>, >=, >>=, >>>=

			    case '>':

					p++;

					if( *p == '=')

					{   p++;

					    t.value = TOK.TOKge;			// >=

					}

					else if( *p == '>')

					{   p++;

					    if( *p == '=')

					    {   p++;

						t.value = TOK.TOKshrass;		// >>=

					    }

					    else if( *p == '>')

					    {	p++;

						if( *p == '=')

						{   p++;

						    t.value = TOK.TOKushrass;	// >>>=

						}

						else

						    t.value = TOK.TOKushr;		// >>>

					    }

					    else

						t.value = TOK.TOKshr;			// >>

					}

					else

					    t.value = TOK.TOKgt;			// >

					return;



			    case '!':

					p++;

					if( *p == '=')

					{   p++;

					    if( *p == '=')

					    {	

							p++;

							t.value = TOK.TOKnotidentity;	// !==

					    }

					    else

						t.value = TOK.TOKnotequal;		// !=

					}

					else if( *p == '<')

					{   

						p++;

					    if( *p == '>')

					    {	

							p++;

							if( *p == '=')

							{  

							 	p++;

						    	t.value = TOK.TOKunord; // !<>=

							}

							else

						  	  t.value = TOK.TOKue;	// !<>

					    }

					    else if( *p == '=')

					    {

							p++;

							t.value = TOK.TOKug;	// !<=

					    }

					    else

							t.value = TOK.TOKuge;	// !<

					}

					else if( *p == '>')

					{   p++;

					    if( *p == '=')

					    {

							p++;

							t.value = TOK.TOKul;	// !>=

					    }

					    else

							t.value = TOK.TOKule;	// !>

					}

					else

					    t.value = TOK.TOKnot;		// !

					return;



			    case '=':

					p++;

					if( *p == '=')

					{   	

						p++;

					    if( *p == '=')

					    {

							p++;

							t.value = TOK.TOKidentity;	// ===

					    }

					    else

							t.value = TOK.TOKequal;		// ==

					}

					else

					    t.value = TOK.TOKassign;		// =

					return;



				case '~':

					p++;

					if( *p == '=' )

					{   

						p++;

						t.value = TOK.TOKcatass;		// ~=

					}

					else

						t.value = TOK.TOKtilde;			// ~

					return;



		// SINGLE

			    case '(': p++; t.value = TOK.TOKlparen; 	return;

			    case ')': p++; t.value = TOK.TOKrparen; 	return;

			    case '[': p++; t.value = TOK.TOKlbracket; 	return;

			    case ']': p++; t.value = TOK.TOKrbracket; 	return;

			    case '{': p++; t.value = TOK.TOKlcurly; 	return;

			    case '}': p++; t.value = TOK.TOKrcurly; 	return;

			    case '?': p++; t.value = TOK.TOKquestion; 	return;

			    case ',': p++; t.value = TOK.TOKcomma; 		return;

			    case ';': p++; t.value = TOK.TOKsemicolon; 	return;

			    case ':': p++; t.value = TOK.TOKcolon; 		return;

			    case '$': p++; t.value = TOK.TOKdollar; 	return;

		// DOUBLE

			    case '*': p++; if( *p == '=' ) { p++; t.value = TOK.TOKmulass; } else t.value = TOK.TOKmul; return;

			    case '%': p++; if( *p == '=' ) { p++; t.value = TOK.TOKmodass; } else t.value = TOK.TOKmod; return;

			    case '^': p++; if( *p == '=' ) { p++; t.value = TOK.TOKxorass; } else t.value = TOK.TOKxor; return;

// removed 148	case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return;





			    case '#':

					p++;

					Pragma();

					continue;

		

			    default:

			    {	

					debug writefln( "    default char" );

					ubyte c = *p;

					if( c & 0x80 )

					{

						uint u = decodeUTF();

					    // Check for start of unicode identifier

					    if( isUniAlpha(u) )

							goto case_ident;



					    if( u == PS || u == LS )

					    {

							loc.linnum++;

							p++;

							continue;

					    }

					}

					if( isprint(c))

					    error("unsupported char '%s'", cast(char)c);

					else

					    error("unsupported char 0x%02x", cast(ubyte)c);

					p++;

					continue;

			    }

			}

	    }

	}







	// Parse escape sequence.

	uint escapeSequence()

	{

		uint c;

	    int n;

	    int ndigits;



	    c = *p;

	    switch ( c )

	    {

			case '\'':

			case '"':

			case '?':

			case '\\':

			Lconsume:

				p++;

				break;



			case 'a':	c = 7;		goto Lconsume;

			case 'b':	c = 8;		goto Lconsume;

			case 'f':	c = 12;		goto Lconsume;

			case 'n':	c = 10;		goto Lconsume;

			case 'r':	c = 13;		goto Lconsume;

			case 't':	c = 9;		goto Lconsume;

			case 'v':	c = 11;		goto Lconsume;

	

			case 'u':

				ndigits = 4;

				goto Lhex;

			case 'U':

				ndigits = 8;

				goto Lhex;

			case 'x':

				ndigits = 2;

			Lhex:

				p++;

				c = *p;

				if( ishex(c))

				{   

					uint v;

				    n = 0;

				    v = 0;

				    while (1)

				    {

						if( isdigit(c))

						    c -= '0';

						else if( islower(c))

						    c -= 'a' - 10;

						else

						    c -= 'A' - 10;

						v = v * 16 + c;

						c = *++p;

						if( ++n == ndigits)

						    break;

						if( !ishex(c))

						{   error("escape hex sequence has %d hex digits instead of %d", n, ndigits);

						    break;

						}

				    }

//!				    if( ndigits != 2 && !utf_isValidDchar(v))

//!						error("invalid UTF character \\U%08x", v);

				    c = v;

				}

				else

				    error("undefined escape hex sequence \\%s\n",c);

				break;



			case '&':			// named character entity

				for( ubyte *idstart = ++p; 1; p++ )

				{

				    switch( *p )

				    {

						case ';':

							//!!!

							/+

						    c = HtmlNamedEntity(idstart, p - idstart);

						    if( c == ~0 )

						    {

								error("unnamed character entity &%.*s;", p - idstart, idstart);

								c = ' ';

						    }



						    p++;

						    +/

						    break;



						default:

						    if( isalpha(*p) || (p != idstart + 1 && isdigit(*p)))

								continue;

						    error("unterminated named entity");

						    break;

				    }

				    break;

				}

				break;



			case 0:

			case 0x1a:			// end of file

				c = '\\';

				break;



			default:

				if( isoctal(c) )

				{   

					ubyte v;

				    n = 0;

				    do

				    {

						v = v * 8 + (c - '0');

						c = *++p;

				    } while (++n < 3 && isoctal(c));

				    c = v;

				}

				else

				    error("undefined escape sequence \\%s\n",c);

				break;

	    }

	    return c;

	}



	/**************************************

	 */



	TOK wysiwygStringConstant( Token *t, int tc )

	{

		uint c;

	    Loc start = loc;



	    p++;

	    stringbuffer.offset = 0;

	    while (1)

	    {

			c = *p++;

			switch( c )

			{

			    case '\n':

					loc.linnum++;

					break;



			    case '\r':

					if( *p == '\n')

					    continue;	// ignore

					c = '\n';	// treat EndOfLine as \n character

					loc.linnum++;

					break;



			    case 0:

			    case 0x1a:

					error("unterminated string constant starting at %s", start.toChars());

					t.ustring = "";

					t.postfix = 0;

					return TOK.TOKstring;



			    case '"':

			    case '`':

				if( c == tc)

				{

//				    t.len = stringbuffer.offset;

				    stringbuffer.write(cast(byte)0);

				    t.ustring = stringbuffer.toString;

//				    t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);

//				    memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);

				    stringPostfix(t);

				    return TOK.TOKstring;

				}

				break;



			    default:

				if( c & 0x80)

				{

					p--;

				    uint u = decodeUTF();

				    p++;

				    if( u == PS || u == LS)

						loc.linnum++;

				    stringbuffer.write(u);

				    continue;

				}

				break;

			}

			stringbuffer.write(c);

	    }

	}



	/**************************************

	 * Lex hex strings:

	 *	x"0A ae 34FE BD"

	 */



	TOK hexStringConstant(Token *t)

	{

		uint c;

	    Loc start = loc;

	    uint n = 0;

	    uint v;

	

	    p++;

	    stringbuffer.offset = 0;

	    while (1)

	    {

			c = *p++;

			switch (c)

			{

			    case ' ':

			    case '\t':

			    case '\v':

			    case '\f':

					continue;			// skip white space



			    case '\r':

					if( *p == '\n')

					    continue;			// ignore

				// Treat isolated '\r' as if it were a '\n'

			    case '\n':

					loc.linnum++;

					continue;



			    case 0:

			    case 0x1a:

					error("unterminated string constant starting at %s", start.toChars());

					t.ustring = "";

					t.postfix = 0;

					return TOK.TOKstring;



			    case '"':

					if( n & 1 )

					{

						error("odd number (%d) of hex characters in hex string", n);

					    stringbuffer.write(v);

					}

//				t.len = stringbuffer.offset;

//				stringbuffer.write(cast(byte)0);

					t.ustring = stringbuffer.toString;

//				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);

//				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);

					stringPostfix(t);

					return TOK.TOKstring;



			    default:

					if( c >= '0' && c <= '9')

					    c -= '0';

					else if( c >= 'a' && c <= 'f')

					    c -= 'a' - 10;

					else if( c >= 'A' && c <= 'F')

					    c -= 'A' - 10;

					else if( c & 0x80)

					{   

						p--;

					    uint u = decodeUTF();

					    p++;

					    if( u == PS || u == LS)

							loc.linnum++;

					    else

							error("non-hex character \\u%x", u);

					}

					else

					    error("non-hex character '%s'", c);

					if( n & 1)

					{

						v = (v << 4) | c;

					    stringbuffer.write(v);

					}

					else

					    v = c;

					n++;

					break;

			}

	    }

	}



	/**************************************

	 */



	TOK escapeStringConstant(Token *t, int wide)

	{

		uint c;

	    Loc start = loc;

	    p++;

	    stringbuffer.offset = 0;

	//    debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) );

	    while( 1 )

	    {

			c = *p++;

			switch( c )

			{

			    case '\\':

					switch (*p)

					{

					    case 'u':

					    case 'U':

					    case '&':

							c = escapeSequence();

							stringbuffer.write(c);

							continue;

	

					    default:

							c = escapeSequence();

							break;

					}

					break;



			    case '\n':

					loc.linnum++;

					break;



			    case '\r':

					if( *p == '\n')

					    continue;	// ignore

					c = '\n';	// treat EndOfLine as \n character

					loc.linnum++;

					break;



			    case '"':

//			    	writefln( "end of string: ", stringbuffer.toString );

					t.ustring = stringbuffer.toString().dup;

	//				t.len = stringbuffer.offset;

	//				stringbuffer.write(cast(byte)0);

	//				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);

	//				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);

					stringPostfix(t);

					

					return TOK.TOKstring;



			    case 0:

			    case 0x1a:

					p--;

					error("unterminated string constant starting at %s", start.toChars());

					t.ustring = "";

//					t.len = 0;

					t.postfix = 0;

					return TOK.TOKstring;



			    default:

					if( c & 0x80 )

					{

					    p--;

					    c = decodeUTF();

					    if( c == LS || c == PS )

					    {

							c = '\n';

							loc.linnum++;

					    }

					    p++;

					    stringbuffer.write(cast(char)c);

					    continue;

					}

					break;

			}

			stringbuffer.write(cast(char)c);

//			writefln( stringbuffer.toString );

	    }

	}



	//**************************************

	TOK charConstant(Token *t, int wide)

	{

		uint c;

	    TOK tk = TOK.TOKcharv;

	    //printf("Lexer.charConstant\n");

	    p++;

	    c = *p++;

	    switch( c )

	    {

			case '\\':

			    switch (*p)

			    {

					case 'u':

					    t.uns64value = escapeSequence();

					    tk = TOK.TOKwcharv;

					    break;



					case 'U':

					case '&':

					    t.uns64value = escapeSequence();

					    tk = TOK.TOKdcharv;

					    break;



					default:

					    t.uns64value = escapeSequence();

					    break;

			    }

			    break;



			case '\n':

			L1:

			    loc.linnum++;

			case '\r':

			case 0:

			case 0x1a:

			case '\'':

			    error("unterminated character constant");

			    return tk;



			default:

			    if( c & 0x80)

			    {

					p--;

					c = decodeUTF();

					p++;

					if( c == LS || c == PS )

					    goto L1;

					if( c < 0xd800 || (c >= 0xe000 && c < 0xfffe))

					    tk = TOK.TOKwcharv;

					else

					    tk = TOK.TOKdcharv;

			    }

			    t.uns64value = c;

			    break;

	    }



	    if( *p != '\'' )

	    {

			error("unterminated character constant");

			return tk;

	    }

	    p++;

	    return tk;

	}



	// Get postfix of string literal.

	void stringPostfix(Token *t)

	{

	    switch( *p )

	    {

			case 'c':

			case 'w':

			case 'd':

			    t.postfix = *p;

			    p++;

			    break;



			default:

			    t.postfix = 0;

			    break;

	    }

	}



	/***************************************

	 * Read \u or \U unicode sequence

	 * Input:

	 *	u	'u' or 'U'

	 */

	/*

	uint Wchar(uint u)

	{

	    uint value;

	    uint n;

	    ubyte c;

	    uint nchars;

	

	    nchars = (u == 'U') ? 8 : 4;

	    value = 0;

	    for (n = 0; 1; n++)

	    {

			++p;

			if( n == nchars)

			    break;

			c = *p;

			if( !ishex(c))

			{

				error("\\%s sequence must be followed by %d hex characters", u, nchars);

			    break;

			}

			if( isdigit(c))

			    c -= '0';

			else if( islower(c))

			    c -= 'a' - 10;

			else

			    c -= 'A' - 10;

			value <<= 4;

			value |= c;

	    }

	    return value;

	}

	*/



	/**************************************

	 * Read in a number.

	 * If it's an integer, store it in tok.TKutok.Vlong.

	 *	integers can be decimal, octal or hex

	 *	Handle the suffixes U, UL, LU, L, etc.

	 * If it's double, store it in tok.TKutok.Vdouble.

	 * Returns:

	 *	TKnum

	 *	TKdouble,...

	 */



	TOK number(Token *t)

	{

	    //debug writefln("Lexer.number()");

	    // We use a state machine to collect numbers

	    enum STATE

		{

			STATE_initial,

			STATE_0,

			STATE_decimal,

			STATE_octal,

			STATE_octale,

			STATE_hex,

			STATE_binary,

			STATE_hex0,

			STATE_binary0,

			STATE_hexh,

			STATE_error

		}



	    enum FLAGS

	    {

			FLAGS_decimal  = 1,		// decimal

			FLAGS_unsigned = 2,		// u or U suffix

			FLAGS_long     = 4,		// l or L suffix

	    }

	    FLAGS flags = FLAGS.FLAGS_decimal;



	    int i;

	    TOK result;

	    int base;

	    stringbuffer.offset = 0;

//		stringbuffer.data = null;

	    STATE state = STATE.STATE_initial;

	    ubyte* start = p;



		TOK _isreal()

		{

			p = start;

			return inreal(t);

		}



	    while( true )

	    {

			char c = cast(char)*p;

			switch( state )

			{

			    case STATE.STATE_initial:		// opening state

					if( c == '0' )

					    state = STATE.STATE_0;

					else

					    state = STATE.STATE_decimal;

					break;



			    case STATE.STATE_0:

					flags = cast(FLAGS) (flags & ~FLAGS.FLAGS_decimal);

					switch (c)

					{

	//	#if ZEROH

//					    case 'H':			// 0h

//					    case 'h':

//							goto hexh;

	//	#endif

					    case 'X':

					    case 'x':

							state = STATE.STATE_hex0;

							break;



					    case '.':

							if( p[1] == '.')	// .. is a separate token

							    goto done;

					    case 'i':

					    case 'f':

					    case 'F':

							goto _Real;

	//	#if ZEROH

//					    case 'E':

//					    case 'e':

//							goto case_hex;

	//	#endif

					    case 'B':

					    case 'b':

							state = STATE.STATE_binary0;

							break;



					    case '0': case '1': case '2': case '3':

					    case '4': case '5': case '6': case '7':

							state = STATE.STATE_octal;

							break;



	//	#if ZEROH

//					    case '8': case '9': case 'A':

//					    case 'C': case 'D': case 'F':

//					    case 'a': case 'c': case 'd': case 'f':

//						    case_hex:

//							state = STATE.STATE_hexh;

//							break;

	//	#endif

					    case '_':

							state = STATE.STATE_octal;

							p++;

							continue;



					    default:

							goto done;

					}

					break;



			    case STATE.STATE_decimal:		// reading decimal number



					// if its not a digit - decimal complete or not a decimal

					if( !isdigit( c ) )

					{

//						debug writefln( "\tnon-digit( %s )", c );

		//	#if ZEROH

//					    if( ishex(c) || c == 'H' || c == 'h' )

//							goto hexh;

		//	#endif

						//! wtf ?

						// ignore embedded _

					    if( c == '_' )

					    {

							p++;

							continue;

					    }



					    // check decimal point - make real

					    if( c == '.' && p[1] != '.' )

							goto _Real;



						// check for mantra - make real

					    if( c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E' )

					    {

					    _Real:	// It's a real number. Back up and rescan as a real

							p = start;

							return inreal(t);

					    }



					    goto done;

					}

					break;



			    case STATE.STATE_hex0:		// reading hex number

			    case STATE.STATE_hex:

					if( !ishex(c) )

					{

					    if( c == '_')		// ignore embedded _

					    {

							p++;

							continue;

					    }

					    if( c == '.' && p[1] != '.' )

							goto _Real;

					    if( c == 'P' || c == 'p' || c == 'i' )

							goto _Real;

					    if( state == STATE.STATE_hex0 )

							error("Hex digit expected, not '%s'", c);

					    goto done;

					}

					state = STATE.STATE_hex;

					break;

	

		//	#if ZEROH

//				    hexh:

//						state = STATE.STATE_hexh;

//

//				    case STATE.STATE_hexh:		// parse numbers like 0FFh

//						if( !ishex(c))

//						{

//						    if( c == 'H' || c == 'h')

//						    {

//								p++;

//								base = 16;

//								goto done;

//						    }

//						    else

//						    {

//								// Check for something like 1E3 or 0E24

//								if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset))

//								    goto _Real;

//								error("Hex digit expected, not '%s'", c);

//								goto done;

//						    }

//						}

//						break;

	//		#endif



				    case STATE.STATE_octal:			// reading octal number

				    case STATE.STATE_octale:		// reading octal number with non-octal digits

						if( !isoctal(c) )

						{

//				#if ZEROH

//						    if( ishex(c) || c == 'H' || c == 'h' )

//								goto hexh;

//				#endif

						    if( c == '_' )		// ignore embedded _

						    {

								p++;

								continue;

						    }

						    if( c == '.' && p[1] != '.' )

								goto _Real;

						    if( c == 'i' )

								goto _Real;

						    if( isdigit(c) )

								state = STATE.STATE_octale;

						    else

								goto done;

						}

						break;



				    case STATE.STATE_binary0:		// starting binary number

				    case STATE.STATE_binary:		// reading binary number

						if( c != '0' && c != '1')

						{

	//			#if ZEROH

//						    if( ishex(c) || c == 'H' || c == 'h' )

//								goto hexh;

	//			#endif

						    if( c == '_' )		// ignore embedded _

						    {

								p++;

								continue;

						    }

						    if( state == STATE.STATE_binary0 )

						    {

								error("binary digit expected");

								state = STATE.STATE_error;

								break;

						    }

						    else

								goto done;

						}

						state = STATE.STATE_binary;

						break;



				    case STATE.STATE_error:		// for error recovery

						if( !isdigit(c) )	// scan until non-digit

						    goto done;

						break;



				    default:

						assert(0);

				}

				stringbuffer.write(cast(char)c);

				p++;

		    }

	done:

	    stringbuffer.write( cast(char)0 );		// terminate string



//		debug writefln( "\tdigit complete( %s )", stringbuffer.toString );



	    if( state == STATE.STATE_octale )

			error("Octal digit expected");



		uinteger_t n;			// unsigned >=64 bit integer type



	    if( stringbuffer.offset == 2 && ( state == STATE.STATE_decimal || state == STATE.STATE_0 ) )

			n = stringbuffer.data[0] - '0';

	    else

	    {

		// Convert string to integer

			char* p = cast(char*)stringbuffer.data.ptr;

			int r = 10;

			int d;

			if( *p == '0' )

			{

			    if( p[1] == 'x' || p[1] == 'X')

				{ 

					// "0x#"

					p += 2; 

					r = 16; 

				}

			    else if( p[1] == 'b' || p[1] == 'B')

				{ 

					// "0b#" - binary

					p += 2; 

					r = 2; 

				}

			    else if( isdigit(p[1]))

				{ 

					p += 1; 

					r = 8; 

				}

			}



			n = 0;



			while( true )

			{

			    if( *p >= '0' && *p <= '9' )

					d = *p - '0';

			    else if( *p >= 'a' && *p <= 'z' )

					d = *p - 'a' + 10;

			    else if( *p >= 'A' && *p <= 'Z' )

					d = *p - 'A' + 10;

			    else

					break;



				if( d >= r )

					break;

			    

				if( n * r + d < n)

			    {

					error( "integer overflow" );

					break;

			    }



			    n = n * r + d;

			    p++;

			}



			// if n needs more than 64 bits

			if( n.sizeof > 8 && n > 0xffffffffffffffffL )

			    error("integer overflow");

	    }



	    // Parse trailing 'u', 'U', 'l' or 'L' in any combination

	    while( true )

	    {

			ubyte f;

			switch( *p )

			{

				case 'U':

			    case 'u':

					f = FLAGS.FLAGS_unsigned;

					goto L1;

			    case 'L':

			    case 'l':

					f = FLAGS.FLAGS_long;

			    L1:

					p++;

					if( flags & f )

					    error("unrecognized token");

					flags = cast(FLAGS) (flags | f);

					continue;

			    default:

					break;

			}

			break;

	    }



	    switch ( flags )

	    {

			case 0:

				/* Octal or Hexadecimal constant.

				 * First that fits: int, uint, long, ulong

				 */

			    if( n & 0x8000000000000000L )

				    result = TOK.TOKuns64v;

			    else if( n & 0xffffffff00000000L )

				    result = TOK.TOKint64v;

			    else if( n & 0x80000000 )

				    result = TOK.TOKuns32v;

			    else

				    result = TOK.TOKint32v;

			    break;



			case FLAGS.FLAGS_decimal:

				/* First that fits: int, long, long long

				 */

			    if( n & 0x8000000000000000L )

			    {

					error("signed integer overflow");

				    result = TOK.TOKuns64v;

			    }

			    else if( n & 0xffffffff80000000L )

				    result = TOK.TOKint64v;

			    else

				    result = TOK.TOKint32v;

			    break;



			case FLAGS.FLAGS_unsigned:

			case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned:

				/* First that fits: uint, ulong

				 */

			    if( n & 0xffffffff00000000L )

				    result = TOK.TOKuns64v;

			    else

				    result = TOK.TOKuns32v;

			    break;



			case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long:

			    if( n & 0x8000000000000000L )

			    {

					error("signed integer overflow");

				    result = TOK.TOKuns64v;

			    }

			    else

				    result = TOK.TOKint64v;

			    break;



			case FLAGS.FLAGS_long:

			    if( n & 0x8000000000000000L )

				    result = TOK.TOKuns64v;

			    else

				    result = TOK.TOKint64v;

			    break;



			case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:

			case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:

			    result = TOK.TOKuns64v;

			    break;



			default:

				debug writefln("%x",flags);

			    assert(0);

	    }

	    t.uns64value = n;

	    return result;

	}

	

	/**************************************

	 * Read in characters, converting them to real.

	 * Bugs:

	 *	Exponent overflow not detected.

	 *	Too much requested precision is not detected.

	 */

	

	TOK inreal(Token *t)

	{

		int dblstate;

	    uint c;

	    char hex;			// is this a hexadecimal-floating-constant?

	    TOK result;



	    //printf("Lexer.inreal()\n");

	    stringbuffer.offset = 0;

	    dblstate = 0;

	    hex = 0;

	Lnext:

	    while (1)

	    {

		// Get next char from input

		c = *p++;

		//printf("dblstate = %d, c = '%s'\n", dblstate, c);

		while (1)

		{

		    switch (dblstate)

		    {

				case 0:			// opening state

				    if( c == '0')

						dblstate = 9;

				    else if( c == '.')

						dblstate = 3;

				    else

						dblstate = 1;

				    break;

	

				case 9:

				    dblstate = 1;

				    if( c == 'X' || c == 'x')

				    {

						hex++;

						break;

				    }

				case 1:			// digits to left of .

				case 3:			// digits to right of .

				case 7:			// continuing exponent digits

				    if( !isdigit(c) && !(hex && isxdigit(c)))

				    {

						if( c == '_')

						    goto Lnext;	// ignore embedded '_'

						dblstate++;

						continue;

				    }

				    break;



				case 2:			// no more digits to left of .

				    if( c == '.')

				    {

						dblstate++;

						break;

				    }

				case 4:			// no more digits to right of .

				    if( (c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p'))

				    {

						dblstate = 5;

						hex = 0;	// exponent is always decimal

						break;

				    }

				    if( hex)

						error("binary-exponent-part required");

				    goto done;



				case 5:			// looking immediately to right of E

				    dblstate++;

				    if( c == '-' || c == '+')

					break;

				case 6:			// 1st exponent digit expected

				    if( !isdigit(c))

					error("exponent expected");

				    dblstate++;

				    break;



				case 8:			// past end of exponent digits

				    goto done;

		    }

		    break;

		}

		stringbuffer.write(c);

	    }

	done:

	    p--;



	    stringbuffer.write(cast(byte)0);



//	#if _WIN32 && __DMC__

	    char *save = __locale_decpoint;

	    __locale_decpoint = ".";

//	#endif

	    t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null);

	    errno = 0;

	    switch( *p )

	    {

			case 'F':

			case 'f':

			    strtof(cast(char *)stringbuffer.data.ptr, null);

			    result = TOK.TOKfloat32v;

			    p++;

			    break;



			default:

			    strtod(cast(char *)stringbuffer.data.ptr, null);

			    result = TOK.TOKfloat64v;

			    break;



			case 'L':

			case 'l':

			    result = TOK.TOKfloat80v;

			    p++;

			    break;

	    }

	    if( *p == 'i' || *p == 'I')

	    {

			p++;

			switch( result )

			{

			    case TOK.TOKfloat32v:

					result = TOK.TOKimaginary32v;

					break;

			    case TOK.TOKfloat64v:

					result = TOK.TOKimaginary64v;

					break;

			    case TOK.TOKfloat80v:

					result = TOK.TOKimaginary80v;

					break;

			}

	    }

//	#if _WIN32 && __DMC__

	    __locale_decpoint = save;

//	#endif

	    if( errno == ERANGE)

			error("number is not representable");

	    return result;

	}









	/*********************************************

	 * Do pragma.

	 * Currently, the only pragma supported is:

	 *	#line linnum [filespec]

	 */



	void Pragma()

	{

	    Token tok;

	    int linnum;

	    char[] filespec;

	    Loc loc = this.loc;



	    scan(&tok);



	    if( tok.value != TOK.TOKidentifier || tok.ident != Id.line )

			goto Lerr;



	    scan(&tok);

	    if( tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v )

			linnum = tok.uns64value - 1;

	    else

			goto Lerr;



	    while (1)

	    {

			switch (*p)

			{

			    case 0:

			    case 0x1a:

			    case '\n':

				    Lnewline:

					this.loc.linnum = linnum;

					if( filespec.length )

					    this.loc.filename = filespec;

					return;



			    case '\r':

					p++;

					if( *p != '\n')

					{   p--;

					    goto Lnewline;

					}

					continue;



			    case ' ':

			    case '\t':

			    case '\v':

			    case '\f':

					p++;

					continue;			// skip white space



			    case '_':

					if( mod && memcmp(p, cast(char*)"__FILE__", 8) == 0)

					{

					    p += 8;

//!					    filespec = mem.strdup(loc.filename ? loc.filename : mod.ident.toChars());

					}

					continue;



			    case '"':

					if( filespec )

					    goto Lerr;

					stringbuffer.offset = 0;

					p++;

					while (1)

					{

						uint c;

					    c = *p;

					    switch (c)

					    {

							case '\n':

							case '\r':

							case 0:

							case 0x1a:

							    goto Lerr;

	

							case '"':

							    stringbuffer.write(cast(byte)0);

	//						    filespec = mem.strdup((char *)stringbuffer.data);

							    filespec = stringbuffer.toString.dup;

							    p++;

							    break;

	

							default:

							    if( c & 0x80 )

							    {

									uint u = decodeUTF();

									if( u == PS || u == LS )

									    goto Lerr;

							    }

							    stringbuffer.write(c);

							    p++;

							    continue;

						    }

						    break;

					}

					continue;



			    default:

					if( *p & 0x80 )

					{   

						uint u = decodeUTF();

					    if( u == PS || u == LS)

						goto Lnewline;

					}

					goto Lerr;

			}

	    }



	Lerr:

	    errorLoc(loc, "#line integer [\"filespec\"]\\n expected");

	}







	/***************************************************

	 * Parse doc comment embedded between t.ptr and p.

	 * Remove trailing blanks and tabs from lines.

	 * Replace all newlines with \n.

	 * Remove leading comment character from each line.

	 * Decide if it's a lineComment or a blockComment.

	 * Append to previous one for this token.

	 */



	void getDocComment( Token *t, uint lineComment )

	{

	    auto OutBuffer buf = new OutBuffer;

	    ubyte ct = t.ptr[2];

 	    ubyte *q = t.ptr + 3;	// start of comment text

	    int linestart = 0;



	    ubyte *qend = p;



	    if( ct == '*' || ct == '+')

			qend -= 2;



	    // Scan over initial row of ****'s or ++++'s or ////'s

	    for (; q < qend; q++)

	    {

			if( *q != ct)

			    break;

	    }



	    // Remove trailing row of ****'s or ++++'s

	    if( ct != '/')

	    {

			for (; q < qend; qend--)

			{

				if( qend[-1] != ct)

					break;

			}

	    }



	    for (; q < qend; q++)

	    {

			ubyte c = *q;



			switch (c)

			{

				case '*':

				case '+':

				if( linestart && c == ct)

				{   linestart = 0;

					// Trim preceding whitespace up to preceding \n

					while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t'))

					buf.offset--;

					continue;

				}

				break;

		

				case ' ':

				case '\t':

				break;



				case '\r':

				if( q[1] == '\n')

					continue;		// skip the \r

				goto Lnewline;

		

				default:

				if( c == 226)

				{

					// If LS or PS

					if( q[1] == 128 &&

					(q[2] == 168 || q[2] == 169))

					{

					q += 2;

					goto Lnewline;

					}

				}

				linestart = 0;

				break;



				Lnewline:

				c = '\n';		// replace all newlines with \n

				case '\n':

				linestart = 1;

		

				// Trim trailing whitespace

				while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t'))

					buf.offset--;

		

				break;

			}

			buf.write(c);

	    }

	

	    // Always end with a newline

	    if( !buf.offset || buf.data[buf.offset - 1] != '\n' )

			buf.writenl();

	

	    //buf.write(cast(char)0);

	

	    // It's a line comment if the start of the doc comment comes

	    // after other non-whitespace on the same line.

//	    ubyte** dc = (lineComment && anyToken)

//				 ? &t.lineComment

//				 : &t.blockComment;

	

	    char[] dc = (lineComment && anyToken) ? t.lineComment : t.blockComment;



	    // Combine with previous doc comment, if any

	    if( dc.length )

			dc = combineComments( dc, buf.toString().dup );

	    else

			dc = buf.toString().dup;

		

//		writefln( dc );



	    if( lineComment && anyToken )

			t.lineComment = dc;

		else

			t.blockComment = dc;



	}



}



// character maps

static ubyte[256] cmtable;



const int CMoctal 	= 0x1;

const int CMhex 	= 0x2;

const int CMidchar 	= 0x4;



ubyte isoctal (ubyte c) { return cmtable[c] & CMoctal; }

ubyte ishex   (ubyte c) { return cmtable[c] & CMhex; }

ubyte isidchar(ubyte c) { return cmtable[c] & CMidchar; }



static void cmtable_init()

{

    for( uint c = 0; c < cmtable.length; c++ )

    {

		if( '0' <= c && c <= '7' )

		    cmtable[c] |= CMoctal;

		if( isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F') )

		    cmtable[c] |= CMhex;

		if( isalnum(c) || c == '_' )

		    cmtable[c] |= CMidchar;

    }

}





/+

struct StringValue

{

    union

    {

		int intvalue;

		void *ptrvalue;

		dchar *string;

    }



    char[] lstring;

}

#define CASE_BASIC_TYPES			

	case TOKwchar: case TOKdchar:		

	case TOKbit: case TOKbool: case TOKchar:		

	case TOKint8: case TOKuns8:		

	case TOKint16: case TOKuns16:		

	case TOKint32: case TOKuns32:		

	case TOKint64: case TOKuns64:		

	case TOKfloat32: case TOKfloat64: case TOKfloat80:		

	case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:	

	case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:	

	case TOKvoid:



#define CASE_BASIC_TYPES_X(t)					\

	case TOKvoid:	 t = Type::tvoid;  goto LabelX;		\

	case TOKint8:	 t = Type::tint8;  goto LabelX;		\

	case TOKuns8:	 t = Type::tuns8;  goto LabelX;		\

	case TOKint16:	 t = Type::tint16; goto LabelX;		\

	case TOKuns16:	 t = Type::tuns16; goto LabelX;		\

	case TOKint32:	 t = Type::tint32; goto LabelX;		\

	case TOKuns32:	 t = Type::tuns32; goto LabelX;		\

	case TOKint64:	 t = Type::tint64; goto LabelX;		\

	case TOKuns64:	 t = Type::tuns64; goto LabelX;		\

	case TOKfloat32: t = Type::tfloat32; goto LabelX;	\

	case TOKfloat64: t = Type::tfloat64; goto LabelX;	\

	case TOKfloat80: t = Type::tfloat80; goto LabelX;	\

	case TOKimaginary32: t = Type::timaginary32; goto LabelX;	\

	case TOKimaginary64: t = Type::timaginary64; goto LabelX;	\

	case TOKimaginary80: t = Type::timaginary80; goto LabelX;	\

	case TOKcomplex32: t = Type::tcomplex32; goto LabelX;	\

	case TOKcomplex64: t = Type::tcomplex64; goto LabelX;	\

	case TOKcomplex80: t = Type::tcomplex80; goto LabelX;	\

	case TOKbit:	 t = Type::tbit;     goto LabelX;	\

	case TOKchar:	 t = Type::tchar;    goto LabelX;	\

	case TOKwchar:	 t = Type::twchar; goto LabelX;	\

	case TOKdchar:	 t = Type::tdchar; goto LabelX;	\

	LabelX

+/