/parsing/d/lexer.d

  1module parsing.d.lexer;
  2
  3import parsing.token;
  4import parsing.lexer;
  5
  6import parsing.d.tokens;
  7
  8import djehuty;
  9
 10import data.stack;
 11
 12import io.console;
 13
 14class DLexer : Lexer {
 15
 16	this(Stream stream) {
 17		super(stream);
 18		_bank = new Stack!(Token);
 19		_stream = stream;
 20	}
 21
 22	void push(Token token) {
 23		_bank.push(token);
 24	}
 25
 26	string line() {
 27		return _line;
 28	}
 29
 30	string line(uint number) {
 31		number--;
 32		if (number >= _lines.length) {
 33			return "";
 34		}
 35		return _lines[number];
 36	}
 37
 38	Token pop() {
 39		if (!_bank.empty) {
 40			return _bank.pop();
 41		}
 42		Token current;
 43		current.line = _lineNumber;
 44		current.column = _pos + 1;
 45
 46		// will give us a string for the line of utf8 characters.
 47		for(;;) {
 48			if (_line is null || _pos >= _line.length) {
 49				if(!_stream.readLine(_line)) {
 50					return Token.init;
 51				}
 52				_lines ~= _line;
 53				_lineNumber++;
 54				_pos = 0;
 55				current.line++;
 56				current.column = 1;
 57			}
 58
 59			// now break up the line into tokens
 60			// the return for the line is whitespace, and can be ignored
 61
 62			for(; _pos <= _line.length; _pos++) {
 63				char chr;
 64				if (_pos == _line.length) {
 65					chr = '\n';
 66				}
 67				else {
 68					chr = _line[_pos];
 69				}
 70				switch (state) {
 71					default:
 72						// error
 73						_error("error");
 74						return Token.init;
 75
 76					case LexerState.Normal:
 77						if (tokenMapping[chr] != DToken.Invalid) {
 78							DToken newType = tokenMapping[chr];
 79							switch(current.type) {
 80								case DToken.And: // &
 81									if (newType == DToken.And) {
 82										// &&
 83										current.type = DToken.LogicalAnd;
 84									}
 85									else if (newType == DToken.Assign) {
 86										// &=
 87										current.type = DToken.AndAssign;
 88									}
 89									else {
 90										goto default;
 91									}
 92									break;
 93								case DToken.Or: // |
 94									if (newType == DToken.Or) {
 95										// ||
 96										current.type = DToken.LogicalOr;
 97									}
 98									else if (newType == DToken.Assign) {
 99										// |=
100										current.type = DToken.OrAssign;
101									}
102									else {
103										goto default;
104									}
105									break;
106								case DToken.Add: // +
107									if (newType == DToken.Assign) {
108										// +=
109										current.type = DToken.AddAssign;
110									}
111									else if (newType == DToken.Add) {
112										// ++
113										current.type = DToken.Increment;
114									}
115									else {
116										goto default;
117									}
118									break;
119								case DToken.Sub: // -
120									if (newType == DToken.Assign) {
121										// -=
122										current.type = DToken.SubAssign;
123									}
124									else if (newType == DToken.Sub) {
125										// --
126										current.type = DToken.Decrement;
127									}
128									else {
129										goto default;
130									}
131									break;
132								case DToken.Div: // /
133									if (newType == DToken.Assign) {
134										// /=
135										current.type = DToken.DivAssign;
136									}
137									else if (newType == DToken.Add) {
138										// /+
139									}
140									else if (newType == DToken.Div) {
141										// //
142									}
143									else if (newType == DToken.Mul) {
144										// /*
145									}
146									else {
147										goto default;
148									}
149									break;
150								case DToken.Mul: // *
151									if (newType == DToken.Assign) {
152										// *=
153										current.type = DToken.MulAssign;
154									}
155									else {
156										goto default;
157									}
158									break;
159								case DToken.Mod: // %
160									if (newType == DToken.Assign) {
161										// %=
162										current.type = DToken.ModAssign;
163									}
164									else {
165										goto default;
166									}
167									break;
168								case DToken.Xor: // ^
169									if (newType == DToken.Assign) {
170										// ^=
171										current.type = DToken.XorAssign;
172									}
173									else {
174										goto default;
175									}
176									break;
177								case DToken.Cat: // ~
178									if (newType == DToken.Assign) {
179										// ~=
180										current.type = DToken.CatAssign;
181									}
182									else {
183										goto default;
184									}
185									break;
186								case DToken.Assign: // =
187									if (newType == DToken.Assign) {
188										// ==
189										current.type = DToken.Equals;
190									}
191									else {
192										goto default;
193									}
194									break;
195								case DToken.LessThan: // <
196									if (newType == DToken.LessThan) {
197										// <<
198										current.type = DToken.ShiftLeft;
199									}
200									else if (newType == DToken.Assign) {
201										// <=
202										current.type = DToken.LessThanEqual;
203									}
204									else if (newType == DToken.GreaterThan) {
205										// <>
206										current.type = DToken.LessThanGreaterThan;
207									}
208									else {
209										goto default;
210									}
211									break;
212								case DToken.GreaterThan: // >
213									if (newType == DToken.GreaterThan) {
214										// >>
215										current.type = DToken.ShiftRight;
216									}
217									else if (newType == DToken.Assign) {
218										// >=
219										current.type = DToken.GreaterThanEqual;
220									}
221									else {
222										goto default;
223									}
224									break;
225								case DToken.ShiftLeft: // <<
226									if (newType == DToken.Assign) {
227										// <<=
228										current.type = DToken.ShiftLeftAssign;
229									}
230									else {
231										goto default;
232									}
233									break;
234								case DToken.ShiftRight: // >>
235									if (newType == DToken.Assign) {
236										// >>=
237										current.type = DToken.ShiftRightAssign;
238									}
239									else if (newType == DToken.GreaterThan) {
240										// >>>
241										current.type = DToken.ShiftRightSigned;
242									}
243									else {
244										goto default;
245									}
246									break;
247								case DToken.ShiftRightSigned: // >>>
248									if (newType == DToken.Assign) {
249										// >>>=
250										current.type = DToken.ShiftRightSignedAssign;
251									}
252									else {
253										goto default;
254									}
255									break;
256								case DToken.LessThanGreaterThan: // <>
257									if (newType == DToken.Assign) {
258										// <>=
259										current.type = DToken.LessThanGreaterThanEqual;
260									}
261									else {
262										goto default;
263									}
264									break;
265								case DToken.Bang: // !
266									if (newType == DToken.LessThan) {
267										// !<
268										current.type = DToken.NotLessThan;
269									}
270									else if (newType == DToken.GreaterThan) {
271										// !>
272										current.type = DToken.NotGreaterThan;
273									}
274									else if (newType == DToken.Assign) {
275										// !=
276										current.type = DToken.NotEquals;
277									}
278									else {
279										goto default;
280									}
281									break;
282								case DToken.NotLessThan: // !<
283									if (newType == DToken.GreaterThan) {
284										// !<>
285										current.type = DToken.NotLessThanGreaterThan;
286									}
287									else if (newType == DToken.Assign) {
288										// !<=
289										current.type = DToken.NotLessThanEqual;
290									}
291									else {
292										goto default;
293									}
294									break;
295								case DToken.NotGreaterThan: // !>
296									if (newType == DToken.Assign) {
297										// !>=
298										current.type = DToken.NotGreaterThanEqual;
299									}
300									else {
301										goto default;
302									}
303									break;
304								case DToken.NotLessThanGreaterThan: // !<>
305									if (newType == DToken.Assign) {
306										// !<>=
307										current.type = DToken.NotLessThanGreaterThanEqual;
308									}
309									else {
310										goto default;
311									}
312									break;
313								case DToken.Dot: // .
314									if (newType == DToken.Dot) {
315										// ..
316										current.type = DToken.Slice;
317									}
318									else {
319										goto default;
320									}
321									break;
322								case DToken.Slice: // ..
323									if (newType == DToken.Dot) {
324										// ...
325										current.type = DToken.Variadic;
326									}
327									else {
328										goto default;
329									}
330									break;
331								case DToken.Invalid:
332									current.type = tokenMapping[chr];
333									break;
334								default:
335									// Token Error
336									if (current.type != DToken.Invalid) {
337										current.columnEnd = _pos;
338										current.lineEnd = _lineNumber;
339										return current;
340									}
341//									_error("Unknown operator.");
342									return Token.init;
343							}
344							
345							continue;
346						}
347
348						// A character that will switch states continues
349
350						// Strings
351						if (chr == '\'') {
352							state = LexerState.String;
353							inStringType = StringType.Character;
354							cur_string = "";
355							if (current.type != DToken.Invalid) {
356								current.columnEnd = _pos;
357								current.lineEnd = _lineNumber;
358								_pos++;
359								return current;
360							}
361							continue;
362						}
363						else if (chr == '"') {
364							state = LexerState.String;
365							inStringType = StringType.DoubleQuote;
366							cur_string = "";
367							if (current.type != DToken.Invalid) {
368								current.columnEnd = _pos;
369								current.lineEnd = _lineNumber;
370								_pos++;
371								return current;
372							}
373							continue;
374						}
375						else if (chr == '`') {
376							state = LexerState.String;
377							inStringType = StringType.WhatYouSeeQuote;
378							cur_string = "";
379							if (current.type != DToken.Invalid) {
380								current.columnEnd = _pos;
381								current.lineEnd = _lineNumber;
382								_pos++;
383								return current;
384							}
385							continue;
386						}
387
388						// Whitespace
389						else if (chr == ' ' || chr == '\t' || chr == '\n') {
390							if (current.type != DToken.Invalid) {
391								current.columnEnd = _pos;
392								current.lineEnd = _lineNumber;
393								_pos++;
394								return current;
395							}
396							current.column++;
397							continue;
398						}
399
400						// Identifiers
401						else if ((chr >= 'a' && chr <= 'z') || (chr >= 'A' && chr <= 'Z') || chr == '_') {
402							state = LexerState.Identifier;
403							cur_string = "";
404							if (current.type != DToken.Invalid) {
405								current.columnEnd = _pos;
406								current.lineEnd = _lineNumber;
407								return current;
408							}
409							goto case LexerState.Identifier;
410						}
411
412						// Numbers
413						else if (chr >= '0' && chr <= '9') {
414							// reset to invalid base
415							cur_base = 0;
416							cur_decimal = 0;
417							cur_denominator = 1;
418							cur_exponent = 0;
419
420							if (current.type == DToken.Dot) {
421								current.type = DToken.Invalid;
422								inDecimal = true;
423								inExponent = false;
424								cur_integer = 0;
425								cur_base = 10;
426								state = LexerState.FloatingPoint;
427								goto case LexerState.FloatingPoint;
428							}
429							else {
430								state = LexerState.Integer;
431
432								if (current.type != DToken.Invalid) {
433									current.columnEnd = _pos;
434									current.lineEnd = _lineNumber;
435									return current;
436								}
437								goto case LexerState.Integer;
438							}
439						}
440						break;
441
442					case LexerState.String:
443						if (inEscape) {
444							inEscape = false;
445							if (chr == 't') {
446								chr = '\t';
447							}
448							else if (chr == 'b') {
449								chr = '\b';
450							}
451							else if (chr == 'r') {
452								chr = '\r';
453							}
454							else if (chr == 'n') {
455								chr = '\n';
456							}
457							else if (chr == '0') {
458								chr = '\0';
459							}
460							else if (chr == 'x' || chr == 'X') {
461								// BLEH!
462							}
463							cur_string ~= chr;
464							continue;
465						}
466
467						if (inStringType == StringType.DoubleQuote) {
468							if (chr == '"') {
469								state = LexerState.Normal;
470								current.type = DToken.StringLiteral;
471								current.columnEnd = _pos;
472								current.lineEnd = _lineNumber;
473								if (cur_string !is null) {
474									current.value = cur_string;
475								}
476								_pos++;
477								return current;
478							}
479						}
480						else if (inStringType == StringType.RawWhatYouSeeQuote) {
481							if (chr == '"') {
482								state = LexerState.Normal;
483								current.type = DToken.StringLiteral;
484								current.columnEnd = _pos;
485								current.lineEnd = _lineNumber;
486								if (cur_string !is null) {
487									current.value = cur_string;
488								}
489								_pos++;
490								return current;
491							}
492						}
493						else if (inStringType == StringType.WhatYouSeeQuote) {
494							if (chr == '`') {
495								state = LexerState.Normal;
496								current.type = DToken.StringLiteral;
497								current.columnEnd = _pos;
498								current.lineEnd = _lineNumber;
499								if (cur_string !is null) {
500									current.value = cur_string;
501								}
502								_pos++;
503								return current;
504							}
505						}
506						else { // StringType.Character
507							if (chr == '\'') {
508								if (cur_string.length > 1) {
509									// error
510									goto default;
511								}
512								state = LexerState.Normal;
513								current.type = DToken.CharacterLiteral;
514								current.columnEnd = _pos;
515								current.lineEnd = _lineNumber;
516								if (cur_string !is null) {
517									current.value = cur_string;
518								}
519								_pos++;
520								return current;
521							}
522						}
523
524						if ((inStringType == StringType.DoubleQuote || inStringType == StringType.Character) && (chr == '\\')) {
525							// Escaped Characters
526							inEscape = true;
527						}
528						else {
529							cur_string ~= chr;
530						}
531						continue;
532					case LexerState.Comment:
533						break;
534					case LexerState.Identifier:
535						// check for valid succeeding character
536						if ((chr < 'a' || chr > 'z') && (chr < 'A' || chr > 'Z') && chr != '_' && (chr < '0' || chr > '9')) {
537							// Invalid identifier symbol
538							static DToken keywordStart = DToken.Abstract;
539							static const string[] keywordList = ["abstract", "alias", "align", "asm", "assert", "auto",
540								"body", "bool", "break", "byte", "case", "cast","catch","cdouble","cent","cfloat","char",
541								"class","const","continue","creal","dchar","debug","default","delegate","delete","deprecated",
542								"do","double","else","enum","export","extern","false","final","finally","float","for","foreach",
543								"foreach_reverse","function","goto","idouble","if","ifloat","import","in","inout","int","interface",
544								"invariant","ireal","is","lazy","long","macro","mixin","module","new","null","out","override",
545								"package","pragma","private","protected","public","real","ref","return","scope","short","static",
546								"struct","super","switch","synchronized","template","this","throw","true","try",
547								"typedef","typeid","typeof","ubyte","ucent","uint","ulong","union","unittest","ushort","version",
548								"void","volatile","wchar","while","with"
549							];
550							current.type = DToken.Identifier;
551
552							foreach(size_t i, keyword; keywordList) {
553								if (cur_string == keyword) {
554									current.type = keywordStart + i;
555									cur_string = null;
556									break;
557								}
558							}
559
560							if (cur_string !is null) {
561								current.value = cur_string;
562							}
563							state = LexerState.Normal;
564							if (current.type != DToken.Invalid) {
565								current.columnEnd = _pos;
566								current.lineEnd = _lineNumber;
567								return current;
568							}
569							goto case LexerState.Normal;
570						}
571						cur_string ~= chr;
572						continue;
573					case LexerState.Integer:
574						// check for valid succeeding character
575
576						// we may want to switch to floating point state
577						if (chr == '.') {
578							if (cur_base <= 0) {
579								cur_base = 10;
580							}
581							else if (cur_base == 2) {
582								_error("Cannot have binary floating point literals");
583								return Token.init;
584							}
585							else if (cur_base == 8) {
586								_error("Cannot have octal floating point literals");
587								return Token.init;
588							}
589
590							// Reset this just in case, it will get interpreted
591							// in the Floating Point state
592							inDecimal = false;
593							inExponent = false;
594
595							state = LexerState.FloatingPoint;
596							goto case LexerState.FloatingPoint;
597						}
598						else if ((chr == 'p' || chr == 'P') && cur_base == 16) {
599							// Reset this just in case, it will get interpreted
600							// in the Floating Point state
601							inDecimal = false;
602							inExponent = false;
603
604							state = LexerState.FloatingPoint;
605							goto case LexerState.FloatingPoint;
606						}
607						else if (chr == '_') {
608							// ignore
609							if (cur_base == -1) {
610								// OCTAL
611								cur_base = 8;
612							}
613						}
614						else if (cur_base == 0) {
615							// this is the first value
616							if (chr == '0') {
617								// octal or 0 or 0.0, etc
618								// use an invalid value so we can decide
619								cur_base = -1;
620								cur_integer = 0;
621							}
622							else if (chr >= '1' && chr <= '9') {
623								cur_base = 10;
624								cur_integer = (chr - '0');
625							}
626							// Cannot be any other value
627							else {
628								_error("Integer literal expected.");
629								return Token.init;
630							}
631						}
632						else if (cur_base == -1) {
633							// this is the second value of an ambiguous base
634							if (chr >= '0' && chr <= '7') {
635								// OCTAL
636								cur_base = 8;
637								cur_integer = (chr - '0');
638							}
639							else if (chr == 'x' || chr == 'X') {
640								// HEX
641								cur_base = 16;
642							}
643							else if (chr == 'b' || chr == 'B') {
644								// BINARY
645								cur_base = 2;
646							}
647							else {
648								// 0 ?
649								current.type = DToken.IntegerLiteral;
650								current.columnEnd = _pos;
651								current.lineEnd = _lineNumber;
652
653								state = LexerState.Normal;
654								return current;
655							}
656						}
657						else if (cur_base == 16) {
658							if ((chr < '0' || chr > '9') && (chr < 'a' || chr > 'f') && (chr < 'A' || chr > 'F')) {
659								current.type = DToken.IntegerLiteral;
660								current.value = cur_integer;
661								current.columnEnd = _pos;
662								current.lineEnd = _lineNumber;
663
664								state = LexerState.Normal;
665								return current;
666							}
667							else {
668								cur_integer *= cur_base;
669								if (chr >= 'a' && chr <= 'f') {
670									cur_integer += 10 + (chr - 'a');
671								}
672								else if (chr >= 'A' && chr <= 'F') {
673									cur_integer += 10 + (chr - 'A');
674								}
675								else {
676									cur_integer += (chr - '0');
677								}
678							}
679						}
680						else if (cur_base == 10) {
681							if (chr < '0' || chr > '9') {
682								current.type = DToken.IntegerLiteral;
683								current.value = cur_integer;
684								current.columnEnd = _pos;
685								current.lineEnd = _lineNumber;
686
687								state = LexerState.Normal;
688								return current;
689							}
690							else {
691								cur_integer *= cur_base;
692								cur_integer += (chr - '0');
693							}
694						}
695						else if (cur_base == 8) {
696							if (chr >= '8' && chr <= '9') {
697								_error("Digits higher than 7 in an octal integer literal are invalid.");
698								return Token.init;
699							}
700							else if (chr < '0' || chr > '7') {
701								current.type = DToken.IntegerLiteral;
702								current.value = cur_integer;
703								current.columnEnd = _pos;
704								current.lineEnd = _lineNumber;
705
706								state = LexerState.Normal;
707								return current;
708							}
709							else {
710								cur_integer *= cur_base;
711								cur_integer += (chr - '0');
712							}
713						}
714						else if (cur_base == 2) {
715							if (chr < '0' || chr > '1') {
716								current.type = DToken.IntegerLiteral;
717								current.value = cur_integer;
718								current.columnEnd = _pos;
719								current.lineEnd = _lineNumber;
720
721								state = LexerState.Normal;
722								return current;
723							}
724							else {
725								cur_integer *= cur_base;
726								cur_integer += (chr - '0');
727							}
728						}
729
730						continue;
731					case LexerState.FloatingPoint:
732						if (chr == '_') {
733							continue;
734						}
735						else if (chr == '.' && (cur_base == 10 || cur_base == 16)) {
736							// We are now parsing the decimal portion
737							if (inDecimal) {
738								_error("Only one decimal point is allowed per floating point literal.");
739								return Token.init;
740							}
741							else if (inExponent) {
742								_error("Cannot put a decimal point after an exponent in a floating point literal.");
743							}
744							inDecimal = true;
745						}
746						else if (cur_base == 16 && (chr == 'p' || chr == 'P')) {
747							// We are now parsing the exponential portion
748							inDecimal = false;
749							inExponent = true;
750							cur_exponent = -1;
751						}
752						else if (cur_base == 10 && (chr == 'e' || chr == 'E')) {
753							// We are now parsing the exponential portion
754							inDecimal = false;
755							inExponent = true;
756							cur_exponent = -1;
757						}
758						else if (cur_base == 10) {
759							if (chr == 'p' || chr == 'P') {
760								_error("Cannot have a hexidecimal exponent in a non-hexidecimal floating point literal.");
761								return Token.init;
762							}
763							else if (chr < '0' || chr > '9') {
764								if (inExponent && cur_exponent == -1) {
765									_error("You need to specify a value for the exponent part of the floating point literal.");
766									return Token.init;
767								}
768								current.type = DToken.FloatingPointLiteral;
769								double value = cast(double)cur_integer + (cast(double)cur_decimal / cast(double)cur_denominator);
770								double exp = 1;
771								for(size_t i = 0; i < cur_exponent; i++) {
772									exp *= cur_base;
773								}
774								value *= exp;
775								current.value = value;
776								current.columnEnd = _pos;
777								current.lineEnd = _lineNumber;
778
779								state = LexerState.Normal;
780								return current;
781							}
782							else if (inExponent) {
783								if (cur_exponent == -1) {
784									cur_exponent = 0;
785								}
786								cur_exponent *= cur_base;
787								cur_exponent += (chr - '0');
788							}
789							else {
790								cur_decimal *= cur_base;
791								cur_denominator *= cur_base;
792								cur_decimal += (chr - '0');
793							}
794						}
795						else { // cur_base == 16
796							if ((chr < '0' || chr > '9') && (chr < 'a' || chr > 'f') && (chr < 'A' || chr > 'F')) {
797								if (inDecimal && !inExponent) {
798									_error("You need to provide an exponent with the decimal portion of a hexidecimal floating point number. Ex: 0xff.3p2");
799									return Token.init;
800								}
801								if (inExponent && cur_exponent == -1) {
802									_error("You need to specify a value for the exponent part of the floating point literal.");
803									return Token.init;
804								}
805								current.type = DToken.FloatingPointLiteral;
806								double value = cast(double)cur_integer + (cast(double)cur_decimal / cast(double)cur_denominator);
807								double exp = 1;
808								for(size_t i = 0; i < cur_exponent; i++) {
809									exp *= 2;
810								}
811								value *= exp;
812								current.value = value;
813								current.columnEnd = _pos;
814								current.lineEnd = _lineNumber;
815
816								state = LexerState.Normal;
817								return current;
818							}
819							else if (inExponent) {
820								if (cur_exponent == -1) {
821									cur_exponent = 0;
822								}
823								cur_exponent *= cur_base;
824								if (chr >= 'A' && chr <= 'F') {
825									cur_exponent += 10 + (chr - 'A');
826								}
827								else if (chr >= 'a' && chr <= 'f') {
828									cur_exponent += 10 + (chr - 'a');
829								}
830								else {
831									cur_exponent += (chr - '0');
832								}
833							}
834							else {
835								cur_decimal *= cur_base;
836								cur_denominator *= cur_base;
837								if (chr >= 'A' && chr <= 'F') {
838									cur_decimal += 10 + (chr - 'A');
839								}
840								else if (chr >= 'a' && chr <= 'f') {
841									cur_decimal += 10 + (chr - 'a');
842								}
843								else {
844									cur_decimal += (chr - '0');
845								}
846							}
847						}
848						continue;
849				}
850			}
851
852			if (current.type != DToken.Invalid) {
853				current.columnEnd = _pos;
854				current.lineEnd = _lineNumber;
855				return current;
856			}
857			current.line++;
858			current.column = 1;
859
860			if (state != LexerState.String) {
861				state = LexerState.Normal;
862			}
863			else {
864				if (inStringType == StringType.Character) {
865					_error("Unmatched character literal.");
866					return Token.init;
867				}
868				cur_string ~= '\n';
869			}
870		}
871
872		return Token.init;
873	}
874
875	int opApply(int delegate(ref Token) loopbody) {
876		int ret;
877
878		Token foo;
879		while((foo = this.pop()).type != DToken.Invalid) {
880			if ((ret = loopbody(foo)) > 0) {
881				return 1;
882			}
883		}
884
885		return ret;
886	}
887
888private:
889
890	void _error(string msg) {
891		Console.forecolor = Color.Red;
892		Console.putln("Lexical Error: file.d @ ", _lineNumber+1, ":", _pos+1, " - ", msg);
893		Console.putln();
894	}
895
896	// Describe the number lexer states
897	enum LexerState : uint {
898		Normal,
899		String,
900		Comment,
901		Identifier,
902		Integer,
903		FloatingPoint
904	}
905
906	LexerState state;
907	bool inEscape;
908
909	// Describe the string lexer states
910	enum StringType : uint {
911		DoubleQuote,		// "..."
912		WhatYouSeeQuote,	// `...`
913		RawWhatYouSeeQuote,	// r"..."
914		Character,			// '.'
915	}
916
917	StringType inStringType;
918
919	// Describe the comment lexer states
920	enum CommentType : uint {
921		BlockComment,
922		LineComment,
923		NestedComment
924	}
925
926	CommentType inCommentType;
927	string cur_string;
928
929	Stream _stream;
930	string _line;
931	size_t _lineNumber;
932	size_t _pos;
933
934	static const DToken[] tokenMapping = [
935		'!':DToken.Bang,
936		':':DToken.Colon,
937		';':DToken.Semicolon,
938		'.':DToken.Dot,
939		',':DToken.Comma,
940		'(':DToken.LeftParen,
941		')':DToken.RightParen,
942		'{':DToken.LeftCurly,
943		'}':DToken.RightCurly,
944		'[':DToken.LeftBracket,
945		']':DToken.RightBracket,
946		'<':DToken.LessThan,
947		'>':DToken.GreaterThan,
948		'=':DToken.Assign,
949		'+':DToken.Add,
950		'-':DToken.Sub,
951		'~':DToken.Cat,
952		'*':DToken.Mul,
953		'/':DToken.Div,
954		'^':DToken.Xor,
955		'|':DToken.Or,
956		'&':DToken.And,
957		'%':DToken.Mod,
958		];
959
960	int cur_base;
961	ulong cur_integer;
962	bool cur_integer_signed;
963	ulong cur_decimal;
964	ulong cur_exponent;
965	ulong cur_denominator;
966	bool inDecimal;
967	bool inExponent;
968
969	string[] _lines;
970
971	Stack!(Token) _bank;
972}