PageRenderTime 15ms CodeModel.GetById 3ms app.highlight 247ms RepoModel.GetById 1ms app.codeStats 1ms

/codemp/icarus/Tokenizer.cpp

https://github.com/Stoiss/jaMME
C++ | 2868 lines | 2621 code | 190 blank | 57 comment | 313 complexity | 305fd183c2937cf8e081fa80a387f557 MD5 | raw file
   1// Tokenizer.cpp
   2#ifndef NOT_USING_MODULES
   3// !!! if you are not using modules, read BELOW !!!
   4#include "module.h" // if you are not using modules, 
   5					// create an empty Module.h in your
   6					// project -- use of modules allows
   7					// the error handler to be overridden
   8					// with a custom CErrHandler
   9#endif
  10#include "tokenizer.h"
  11
  12#pragma warning(disable : 4100) //unreferenced formal parameter
  13#pragma warning(disable : 4127) //conditional expression is constant
  14#pragma warning(disable : 4189) //local variable is initialized but not referenced
  15#pragma warning(disable : 4244) //conversion from x to x, possible loss of data
  16
  17#ifndef _WIN32
  18#include <stdio.h>
  19#include <stdlib.h>
  20#endif
  21
  22enum
  23{
  24	DIR_INCLUDE = TK_USERDEF,
  25	DIR_IFDEF,
  26	DIR_IFNDEF,
  27	DIR_ENDIF,
  28	DIR_ELSE,
  29	DIR_DEFINE,
  30	DIR_UNDEFINE,
  31};
  32
  33keywordArray_t CTokenizer::directiveKeywords[] =
  34{
  35	"include",		DIR_INCLUDE,
  36	"ifdef",		DIR_IFDEF,
  37	"ifndef",		DIR_IFNDEF,
  38	"endif",		DIR_ENDIF,
  39	"else",			DIR_ELSE,
  40	"define",		DIR_DEFINE,
  41	"undefine",		DIR_UNDEFINE,
  42	"",				TK_EOF,
  43};
  44
  45keywordArray_t CTokenizer::errorMessages[] =
  46{
  47	"No Error",							TKERR_NONE,
  48	"Unknown Error",					TKERR_UNKNOWN,
  49	"Buffer creation failed",			TKERR_BUFFERCREATE,
  50	"Unrecognized symbol",				TKERR_UNRECOGNIZEDSYMBOL,
  51	"Duplicate symbol",					TKERR_DUPLICATESYMBOL,
  52	"String length exceeded",			TKERR_STRINGLENGTHEXCEEDED,
  53	"Identifier length exceeded",		TKERR_IDENTIFIERLENGTHEXCEEDED,
  54	"Expected integer",					TKERR_EXPECTED_INTEGER,
  55	"Expected identifier",				TKERR_EXPECTED_IDENTIFIER,
  56	"Expected string",					TKERR_EXPECTED_STRING,
  57	"Expected char",					TKERR_EXPECTED_CHAR,
  58	"Expected float",					TKERR_EXPECTED_FLOAT,
  59	"Unexpected token",					TKERR_UNEXPECTED_TOKEN,
  60	"Invalid directive",				TKERR_INVALID_DIRECTIVE,
  61	"Include file not found",			TKERR_INCLUDE_FILE_NOTFOUND,
  62	"Unmatched directive",				TKERR_UNMATCHED_DIRECTIVE,
  63	"",									TKERR_USERERROR,
  64};
  65
  66//
  67// CSymbol
  68//
  69
  70CSymbol::CSymbol()
  71{
  72}
  73
  74CSymbol::~CSymbol()
  75{
  76}
  77
  78CSymbol* CSymbol::Create(LPCTSTR symbolName)
  79{
  80	CSymbol* retval = new CSymbol();
  81	retval->Init(symbolName);
  82	return retval;
  83}
  84
  85LPCTSTR CSymbol::GetName()
  86{
  87	if (m_symbolName == NULL)
  88	{
  89		return "";
  90	}
  91	return m_symbolName;
  92}
  93
  94void CSymbol::Init(LPCTSTR symbolName)
  95{
  96	m_symbolName = (char*)malloc(strlen(symbolName) + 1);
  97//	ASSERT(m_symbolName);
  98	strcpy(m_symbolName, symbolName);
  99}
 100
 101void CSymbol::Delete()
 102{
 103	if (m_symbolName != NULL)
 104	{
 105		free(m_symbolName);
 106		m_symbolName = NULL;
 107	}
 108	delete this;
 109}
 110
 111//
 112// CDirectiveSymbol
 113//
 114
 115CDirectiveSymbol::CDirectiveSymbol()
 116{
 117}
 118
 119CDirectiveSymbol::~CDirectiveSymbol()
 120{
 121}
 122
 123CDirectiveSymbol* CDirectiveSymbol::Create(LPCTSTR symbolName)
 124{
 125	CDirectiveSymbol* retval = new CDirectiveSymbol();
 126	retval->Init(symbolName);
 127	return retval;
 128}
 129
 130void CDirectiveSymbol::Init(LPCTSTR symbolName)
 131{
 132	CSymbol::Init(symbolName);
 133	m_value = NULL;
 134}
 135
 136void CDirectiveSymbol::Delete()
 137{
 138	if (m_value != NULL)
 139	{
 140		free(m_value);
 141		m_value = NULL;
 142	}
 143	CSymbol::Delete();
 144}
 145
 146void CDirectiveSymbol::SetValue(LPCTSTR value)
 147{
 148	if (m_value != NULL)
 149	{
 150		free(m_value);
 151	}
 152	m_value = (char*)malloc(strlen(value) + 1);
 153	strcpy(m_value, value);
 154}
 155
 156LPCTSTR CDirectiveSymbol::GetValue()
 157{
 158	return m_value;
 159}
 160
 161//
 162// CIntSymbol
 163//
 164
 165CIntSymbol::CIntSymbol()
 166{
 167}
 168
 169CIntSymbol* CIntSymbol::Create(LPCTSTR symbolName, int value)
 170{
 171	CIntSymbol* retval = new CIntSymbol();
 172	retval->Init(symbolName, value);
 173	return retval;
 174}
 175
 176void CIntSymbol::Delete()
 177{
 178	CSymbol::Delete();
 179}
 180
 181void CIntSymbol::Init(LPCTSTR symbolName, int value)
 182{
 183	CSymbol::Init(symbolName);
 184	m_value = value;
 185}
 186
 187int CIntSymbol::GetValue()
 188{
 189	return m_value;
 190}
 191
 192//
 193// CSymbolTable
 194//
 195
 196CSymbolTable::CSymbolTable()
 197{
 198	Init();
 199}
 200
 201CSymbolTable::~CSymbolTable()
 202{
 203}
 204
 205CSymbolTable* CSymbolTable::Create()
 206{
 207	CSymbolTable* retval = new CSymbolTable();
 208	retval->Init();
 209	return retval;
 210}
 211
 212void CSymbolTable::Init()
 213{
 214}
 215
 216void CSymbolTable::DiscardSymbols()
 217{
 218	for (symbolmap_t::iterator isymbol = m_symbols.begin(); isymbol != m_symbols.end(); isymbol++)
 219	{
 220		(*isymbol).second->Delete();
 221	}
 222	m_symbols.erase(m_symbols.begin(), m_symbols.end());
 223}
 224
 225void CSymbolTable::Delete()
 226{
 227	DiscardSymbols();
 228	delete this;
 229}
 230
 231bool CSymbolTable::AddSymbol(CSymbol* theSymbol)
 232{
 233	LPCTSTR name = theSymbol->GetName();
 234	
 235	symbolmap_t::iterator iter = m_symbols.find(name);
 236	if (iter != m_symbols.end())
 237	{
 238		return false;
 239	}
 240	m_symbols.insert(symbolmap_t::value_type(name, theSymbol));
 241	return true;
 242}
 243
 244CSymbol* CSymbolTable::FindSymbol(LPCTSTR symbolName)
 245{
 246	symbolmap_t::iterator iter = m_symbols.find(symbolName);
 247	if (iter != m_symbols.end())
 248	{
 249		return (*iter).second;
 250	}
 251	return NULL;
 252}
 253
 254CSymbol* CSymbolTable::ExtractSymbol(LPCTSTR symbolName)
 255{
 256	symbolmap_t::iterator iter = m_symbols.find(symbolName);
 257	if (iter != m_symbols.end())
 258	{
 259		CSymbol* retval = (*iter).second;
 260		m_symbols.erase(iter);
 261	}
 262	return NULL;
 263}
 264
 265void CSymbolTable::RemoveSymbol(LPCTSTR symbolName)
 266{
 267	m_symbols.erase(symbolName);
 268}
 269
 270//
 271// CParseStream
 272//
 273
 274CParseStream::CParseStream()
 275{
 276}
 277
 278CParseStream::~CParseStream()
 279{
 280}
 281
 282CParseStream* CParseStream::Create()
 283{
 284	return NULL;
 285}
 286
 287void CParseStream::Delete()
 288{
 289	delete this;
 290}
 291
 292bool CParseStream::Init()
 293{
 294	m_next = NULL;
 295
 296	return true;
 297}
 298
 299bool CParseStream::NextChar(byte& theByte)
 300{
 301	return false;
 302}
 303
 304long CParseStream::GetRemainingSize()
 305{
 306	return 0;
 307}
 308
 309CParseStream* CParseStream::GetNext()
 310{
 311	return m_next;
 312}
 313
 314void CParseStream::SetNext(CParseStream* next)
 315{
 316	m_next = next;
 317}
 318
 319int CParseStream::GetCurLine()
 320{
 321	return 0;
 322}
 323
 324void CParseStream::GetCurFilename(char** theBuff)
 325{
 326	*theBuff = NULL;
 327}
 328
 329bool CParseStream::IsThisDefinition(void* theDefinition)
 330{
 331	return false;
 332}
 333
 334//
 335// CParsePutBack
 336//
 337
 338CParsePutBack::CParsePutBack()
 339{
 340}
 341
 342CParsePutBack::~CParsePutBack()
 343{
 344}
 345
 346CParsePutBack* CParsePutBack::Create(byte theByte, int curLine, LPCTSTR filename)
 347{
 348	CParsePutBack* curParsePutBack = new CParsePutBack();
 349	curParsePutBack->Init(theByte, curLine, filename);
 350	return curParsePutBack;
 351}
 352
 353void CParsePutBack::Delete()
 354{
 355	if (m_curFile != NULL)
 356	{
 357		free(m_curFile);
 358		m_curFile = NULL;
 359	}
 360	delete this;
 361}
 362
 363bool CParsePutBack::NextChar(byte& theByte)
 364{
 365	if (m_consumed)
 366	{
 367		return false;
 368	}
 369	theByte = m_byte;
 370	m_consumed = true;
 371	return true;
 372}
 373
 374void CParsePutBack::Init(byte theByte, int curLine, LPCTSTR filename)
 375{
 376	CParseStream::Init();
 377	m_consumed = false;
 378	m_byte = theByte;
 379	m_curLine = curLine;
 380	if (filename != NULL)
 381	{
 382		m_curFile = (char*)malloc(strlen(filename) + 1);
 383		strcpy(m_curFile, filename);
 384	}
 385	else 
 386	{
 387		m_curFile = NULL;
 388	}
 389}
 390
 391long CParsePutBack::GetRemainingSize()
 392{
 393	if (m_consumed)
 394	{
 395		return 0;
 396	}
 397	else
 398	{
 399		return 1;
 400	}
 401}
 402
 403int CParsePutBack::GetCurLine()
 404{
 405	return m_curLine;
 406}
 407
 408void CParsePutBack::GetCurFilename(char** theBuff)
 409{
 410	if (m_curFile == NULL)
 411	{
 412		*theBuff = NULL;
 413		return;
 414	}
 415	*theBuff = (char*)malloc(strlen(m_curFile) + 1);
 416	strcpy(*theBuff, m_curFile);
 417}
 418
 419//
 420// CParseFile
 421//
 422
 423CParseFile::CParseFile()
 424{
 425}
 426
 427CParseFile::~CParseFile()
 428{
 429}
 430
 431CParseFile* CParseFile::Create()
 432{
 433	CParseFile* theParseFile = new CParseFile();
 434	
 435	if ( !theParseFile->Init() )
 436	{
 437		delete theParseFile;
 438		return NULL;
 439	}
 440
 441	return theParseFile;
 442}
 443
 444CParseFile* CParseFile::Create(LPCTSTR filename, CTokenizer* tokenizer)
 445{
 446	CParseFile* theParseFile = new CParseFile();
 447	
 448	if ( theParseFile->Init(filename, tokenizer) )
 449		return theParseFile;
 450
 451	return NULL;
 452}
 453
 454void CParseFile::Delete()
 455{
 456	if (m_buff != NULL)
 457	{
 458		free(m_buff);
 459		m_buff = NULL;
 460	}
 461	if (m_ownsFile && (m_fileHandle != NULL))
 462	{
 463#ifdef _WIN32
 464		CloseHandle(m_fileHandle);
 465#else
 466		fclose(m_fileHandle);
 467#endif
 468		m_fileHandle = NULL;
 469	}
 470	if (m_fileName != NULL)
 471	{
 472		free(m_fileName);
 473		m_fileName = NULL;
 474	}
 475	delete this;
 476}
 477
 478bool CParseFile::Init()
 479{
 480	m_fileHandle = NULL;
 481	m_buff = NULL;
 482	m_ownsFile = false;
 483	m_curByte = NULL;
 484	m_curLine = 1;
 485	m_fileName = NULL;
 486	return CParseStream::Init();
 487}
 488
 489DWORD CParseFile::GetFileSize()
 490{
 491#ifdef _WIN32
 492	DWORD dwCur = SetFilePointer(m_fileHandle, 0L, NULL, FILE_CURRENT);
 493	DWORD dwLen = SetFilePointer(m_fileHandle, 0, NULL, FILE_END);
 494	SetFilePointer(m_fileHandle, dwCur, NULL, FILE_BEGIN);
 495#else
 496	fseek(m_fileHandle, 0L, SEEK_END);
 497	DWORD dwLen = ftell(m_fileHandle);
 498	fseek(m_fileHandle, 0L, SEEK_SET);
 499#endif
 500	return dwLen;
 501}
 502
 503void CParseFile::Read(void* buff, UINT buffsize)
 504{
 505	DWORD bytesRead;
 506#ifdef _WIN32
 507	ReadFile(m_fileHandle, buff, buffsize, &bytesRead, NULL);
 508#else
 509	fread(buff, buffsize, 1, m_fileHandle);
 510#endif
 511}
 512
 513bool CParseFile::Init(LPCTSTR filename, CTokenizer* tokenizer)
 514{
 515	CParseStream::Init();
 516	m_fileName = (char*)malloc(strlen(filename) + 1);
 517	strcpy(m_fileName, filename);
 518
 519#ifdef _WIN32
 520		DWORD dwAccess = GENERIC_READ;
 521		DWORD dwShareMode = FILE_SHARE_WRITE | FILE_SHARE_READ;
 522		SECURITY_ATTRIBUTES sa;
 523		sa.nLength = sizeof(sa);
 524		sa.lpSecurityDescriptor = NULL;
 525		sa.bInheritHandle = 0;
 526		DWORD dwCreateFlag = OPEN_EXISTING;
 527
 528		m_fileHandle = CreateFile(filename, dwAccess, dwShareMode, &sa, dwCreateFlag, FILE_ATTRIBUTE_NORMAL, NULL);
 529
 530		if (m_fileHandle == (HANDLE)-1)
 531		{
 532			tokenizer->Error(TKERR_INCLUDE_FILE_NOTFOUND);
 533			Init();
 534
 535			return false;
 536		}
 537#else
 538		m_fileHandle = fopen(filename, "r+");
 539
 540		if (m_fileHandle == NULL)
 541		{
 542			tokenizer->Error(TKERR_INCLUDE_FILE_NOTFOUND);
 543			Init();
 544
 545			return false;
 546		}
 547#endif
 548		m_filesize = GetFileSize();
 549		m_buff = (byte*)malloc(m_filesize);
 550		if (m_buff == NULL)
 551		{
 552			tokenizer->Error(TKERR_BUFFERCREATE);
 553			Init();
 554			return false;
 555		}
 556		Read(m_buff, m_filesize);
 557		m_curByte = 0;
 558		m_curPos = 1;
 559		m_ownsFile = true;
 560		m_curLine = 1;
 561
 562	return true;
 563}
 564
 565long CParseFile::GetRemainingSize()
 566{
 567	return m_filesize - m_curByte;
 568}
 569
 570bool CParseFile::NextChar(byte& theByte)
 571{
 572	if (m_curByte < m_filesize)
 573	{
 574		if (m_buff[m_curByte] == '\n')
 575		{
 576			m_curLine += 1;
 577			m_curPos = 1;
 578		}
 579		else
 580		{
 581			m_curPos++;
 582		}
 583		theByte = m_buff[m_curByte++];
 584		return true;
 585	}
 586	else
 587	{
 588		return false;
 589	}
 590}
 591
 592int CParseFile::GetCurLine()
 593{
 594	return m_curLine;
 595}
 596
 597void CParseFile::GetCurFilename(char** theBuff)
 598{
 599	*theBuff = NULL;
 600	if (m_fileName != NULL)
 601	{
 602		*theBuff = (char*)malloc(strlen(m_fileName) + 1);
 603		strcpy(*theBuff, m_fileName);
 604	}
 605}
 606
 607//
 608// CParseMemory
 609//
 610
 611CParseMemory::CParseMemory()
 612{
 613}
 614
 615CParseMemory::~CParseMemory()
 616{
 617}
 618
 619CParseMemory* CParseMemory::Create(byte* data, long datasize)
 620{
 621	CParseMemory* curParse = new CParseMemory();
 622	curParse->Init(data, datasize);
 623	return curParse;
 624}
 625
 626void CParseMemory::Delete()
 627{
 628	delete this;
 629}
 630
 631bool CParseMemory::NextChar(byte& theByte)
 632{
 633	if (m_offset < m_datasize)
 634	{
 635		if (m_data[m_offset] == '\n')
 636		{
 637			m_curLine += 1;
 638			m_curPos = 1;
 639		}
 640		else
 641		{
 642			m_curPos++;
 643		}
 644		theByte = m_data[m_offset++];
 645		return true;
 646	}
 647	else
 648	{
 649		return false;
 650	}
 651}
 652
 653void CParseMemory::Init(byte* data, long datasize)
 654{
 655	m_data = data;
 656	m_curLine = 1;
 657	m_curPos = 1;
 658	m_offset = 0;
 659	m_datasize = datasize;
 660}
 661
 662long CParseMemory::GetRemainingSize()
 663{
 664	return m_datasize - m_offset;
 665}
 666
 667int CParseMemory::GetCurLine()
 668{
 669	return m_curLine;
 670}
 671
 672void CParseMemory::GetCurFilename(char** theBuff)
 673{
 674	*theBuff = NULL;
 675}
 676
 677//
 678// CParseBlock
 679//
 680
 681CParseBlock::CParseBlock()
 682{
 683}
 684
 685CParseBlock::~CParseBlock()
 686{
 687}
 688
 689CParseBlock* CParseBlock::Create(byte* data, long datasize)
 690{
 691	CParseBlock* curParse = new CParseBlock();
 692	curParse->Init(data, datasize);
 693	return curParse;
 694}
 695
 696void CParseBlock::Delete()
 697{
 698	if (m_data != NULL)
 699	{
 700		free(m_data);
 701		m_data = NULL;
 702	}
 703	delete this;
 704}
 705
 706void CParseBlock::Init(byte* data, long datasize)
 707{
 708	m_data = (byte*)malloc(datasize);
 709	memcpy(m_data, data, datasize);
 710	m_curLine = 1;
 711	m_curPos = 1;
 712	m_offset = 0;
 713	m_datasize = datasize;
 714}
 715
 716//
 717// CParseToken
 718//
 719
 720CParseToken::CParseToken()
 721{
 722}
 723
 724CParseToken::~CParseToken()
 725{
 726}
 727
 728CParseToken* CParseToken::Create(CToken* token)
 729{
 730	CParseToken* curParse = new CParseToken();
 731	curParse->Init(token);
 732	return curParse;
 733}
 734
 735void CParseToken::Delete()
 736{
 737	if (m_data != NULL)
 738	{
 739		free(m_data);
 740		m_data = NULL;
 741	}
 742	delete this;
 743}
 744
 745bool CParseToken::NextChar(byte& theByte)
 746{
 747	if (m_offset < m_datasize)
 748	{
 749		if (m_data[m_offset] == '\n')
 750		{
 751			m_curLine += 1;
 752			m_curPos = 1;
 753		}
 754		else
 755		{
 756			m_curPos++;
 757		}
 758		theByte = m_data[m_offset++];
 759		return true;
 760	}
 761	else
 762	{
 763		return false;
 764	}
 765}
 766
 767void CParseToken::Init(CToken* token)
 768{
 769	LPCTSTR tokenString = token->GetStringValue();
 770	m_datasize = strlen(tokenString);
 771	if (m_datasize > 0)
 772	{
 773		m_data = (byte*) malloc(m_datasize);
 774		memcpy(m_data, tokenString, m_datasize);
 775	}
 776	else
 777	{
 778		m_data = NULL;
 779	}
 780	m_curLine = 1;
 781	m_curPos = 1;
 782	m_offset = 0;
 783	token->Delete();
 784}
 785
 786long CParseToken::GetRemainingSize()
 787{
 788	return m_datasize - m_offset;
 789}
 790
 791int CParseToken::GetCurLine()
 792{
 793	return m_curLine;
 794}
 795
 796void CParseToken::GetCurFilename(char** theBuff)
 797{
 798	*theBuff = NULL;
 799}
 800
 801//
 802// CParseDefine
 803//
 804
 805CParseDefine::CParseDefine()
 806{
 807}
 808
 809CParseDefine::~CParseDefine()
 810{
 811}
 812
 813CParseDefine* CParseDefine::Create(CDirectiveSymbol* definesymbol)
 814{
 815	CParseDefine* retval = new CParseDefine();
 816	retval->Init(definesymbol);
 817	return retval;
 818}
 819
 820void CParseDefine::Delete()
 821{
 822	CParseMemory::Delete();
 823}
 824
 825void CParseDefine::Init(CDirectiveSymbol* definesymbol)
 826{
 827	CParseMemory::Init((byte*)definesymbol->GetValue(), strlen(definesymbol->GetValue()));
 828	m_defineSymbol = definesymbol;
 829}
 830
 831bool CParseDefine::IsThisDefinition(void* theDefinition)
 832{
 833	return (CDirectiveSymbol*)theDefinition == m_defineSymbol;
 834}
 835
 836//
 837// CToken
 838//
 839
 840CToken::CToken()
 841{
 842}
 843
 844CToken::~CToken()
 845{
 846}
 847
 848CToken* CToken::Create()
 849{
 850	CToken* theToken = new CToken();
 851	theToken->Init();
 852	return theToken;
 853}
 854
 855void CToken::Delete()
 856{
 857	if (m_string != NULL)
 858	{
 859		free(m_string);
 860		m_string = NULL;
 861	}
 862	delete this;
 863}
 864
 865void CToken::Init()
 866{
 867	m_next = NULL;
 868	m_string = NULL;
 869}
 870
 871void CToken::SetNext(CToken* theToken)
 872{
 873	m_next = theToken;
 874}
 875
 876CToken* CToken::GetNext()
 877{
 878	return m_next;
 879}
 880
 881int CToken::GetType()
 882{
 883	return TK_EOF;
 884}
 885
 886int CToken::GetIntValue()
 887{
 888	return 0;
 889}
 890
 891LPCTSTR CToken::GetStringValue()
 892{
 893	if (m_string == NULL)
 894	{
 895		return "";
 896	}
 897	return m_string;
 898}
 899
 900float CToken::GetFloatValue()
 901{
 902	return 0.0;
 903}
 904
 905//
 906// CCharToken
 907//
 908
 909CCharToken::CCharToken()
 910{
 911}
 912
 913CCharToken::~CCharToken()
 914{
 915}
 916
 917CCharToken* CCharToken::Create(byte theByte)
 918{
 919	CCharToken* theToken = new CCharToken();
 920	theToken->Init(theByte);
 921	return theToken;
 922}
 923
 924void CCharToken::Delete()
 925{
 926	CToken::Delete();
 927}
 928
 929void CCharToken::Init(byte theByte)
 930{
 931	CToken::Init();
 932	char charString[10];
 933	switch(theByte)
 934	{
 935		case '\0':
 936			strcpy(charString, "\\0");
 937			break;
 938		case '\n':
 939			strcpy(charString, "\\n");
 940			break;
 941		case '\\':
 942			strcpy(charString, "\\\\");
 943			break;
 944		case '\'':
 945			strcpy(charString, "\\'");
 946			break;
 947		case '\?':
 948			strcpy(charString, "\\?");
 949			break;
 950		case '\a':
 951			strcpy(charString, "\\a");
 952			break;
 953		case '\b':
 954			strcpy(charString, "\\b");
 955			break;
 956		case '\f':
 957			strcpy(charString, "\\f");
 958			break;
 959		case '\r':
 960			strcpy(charString, "\\r");
 961			break;
 962		case '\t':
 963			strcpy(charString, "\\t");
 964			break;
 965		case '\v':
 966			strcpy(charString, "\\v");
 967			break;
 968		default:
 969			charString[0] = (char)theByte;
 970			charString[1] = '\0';
 971			break;
 972	}
 973	m_string = (char*)malloc(strlen(charString) + 1);
 974	strcpy(m_string, charString);
 975}
 976
 977int CCharToken::GetType()
 978{
 979	return TK_CHAR;
 980}
 981
 982//
 983// CStringToken
 984//
 985
 986CStringToken::CStringToken()
 987{
 988}
 989
 990CStringToken::~CStringToken()
 991{
 992}
 993
 994CStringToken* CStringToken::Create(LPCTSTR theString)
 995{
 996	CStringToken* theToken = new CStringToken();
 997	theToken->Init(theString);
 998	return theToken;
 999}
1000
1001void CStringToken::Delete()
1002{
1003	CToken::Delete();
1004}
1005
1006void CStringToken::Init(LPCTSTR theString)
1007{
1008	CToken::Init();
1009	m_string = (char*)malloc(strlen(theString) + 1);
1010//	ASSERT(m_string);
1011	strcpy(m_string, theString);
1012}
1013
1014int CStringToken::GetType()
1015{
1016	return TK_STRING;
1017}
1018
1019//
1020// CIntToken
1021//
1022
1023CIntToken::CIntToken()
1024{
1025}
1026
1027CIntToken::~CIntToken()
1028{
1029}
1030
1031CIntToken* CIntToken::Create(long value)
1032{
1033	CIntToken* theToken = new CIntToken();
1034	theToken->Init(value);
1035	return theToken;
1036}
1037
1038void CIntToken::Delete()
1039{
1040	CToken::Delete();
1041}
1042
1043void CIntToken::Init(long value)
1044{
1045	CToken::Init();
1046	m_value = value;
1047}
1048
1049int CIntToken::GetType()
1050{
1051	return TK_INT;
1052}
1053
1054int CIntToken::GetIntValue()
1055{
1056	return m_value;
1057}
1058
1059float CIntToken::GetFloatValue()
1060{
1061	return (float)m_value;
1062}
1063
1064LPCTSTR CIntToken::GetStringValue()
1065{
1066	if (m_string != NULL)
1067	{
1068		free(m_string);
1069		m_string = NULL;
1070	}
1071	char temp[128];
1072	sprintf(temp, "%d", m_value);
1073	m_string = (char*)malloc(strlen(temp) + 1);
1074	strcpy(m_string, temp);
1075	return m_string;
1076}
1077
1078//
1079// CFloatToken
1080//
1081
1082CFloatToken::CFloatToken()
1083{
1084}
1085
1086CFloatToken::~CFloatToken()
1087{
1088}
1089
1090CFloatToken* CFloatToken::Create(float value)
1091{
1092	CFloatToken* theToken = new CFloatToken();
1093	theToken->Init(value);
1094	return theToken;
1095}
1096
1097void CFloatToken::Delete()
1098{
1099	CToken::Delete();
1100}
1101
1102void CFloatToken::Init(float value)
1103{
1104	CToken::Init();
1105	m_value = value;
1106}
1107
1108int CFloatToken::GetType()
1109{
1110	return TK_FLOAT;
1111}
1112
1113float CFloatToken::GetFloatValue()
1114{
1115	return m_value;
1116}
1117
1118LPCTSTR CFloatToken::GetStringValue()
1119{
1120	if (m_string != NULL)
1121	{
1122		free(m_string);
1123		m_string = NULL;
1124	}
1125	char temp[128];
1126	sprintf(temp, "%g", m_value);
1127	m_string = (char*)malloc(strlen(temp) + 1);
1128	strcpy(m_string, temp);
1129	return m_string;
1130}
1131
1132//
1133// CIdentifierToken
1134//
1135
1136CIdentifierToken::CIdentifierToken()
1137{
1138}
1139
1140CIdentifierToken::~CIdentifierToken()
1141{
1142}
1143
1144CIdentifierToken* CIdentifierToken::Create(LPCTSTR name)
1145{
1146	CIdentifierToken* theToken = new CIdentifierToken();
1147	theToken->Init(name);
1148	return theToken;
1149}
1150
1151void CIdentifierToken::Delete()
1152{
1153	CToken::Delete();
1154}
1155
1156void CIdentifierToken::Init(LPCTSTR name)
1157{
1158	CToken::Init();
1159	m_string = (char*)malloc(strlen(name) + 1);
1160//	ASSERT(m_string);
1161	strcpy(m_string, name);
1162}
1163
1164int CIdentifierToken::GetType()
1165{
1166	return TK_IDENTIFIER;
1167}
1168
1169//
1170// CCommentToken
1171//
1172
1173CCommentToken::CCommentToken()
1174{
1175}
1176
1177CCommentToken::~CCommentToken()
1178{
1179}
1180
1181CCommentToken* CCommentToken::Create(LPCTSTR name)
1182{
1183	CCommentToken* theToken = new CCommentToken();
1184	theToken->Init(name);
1185	return theToken;
1186}
1187
1188void CCommentToken::Delete()
1189{
1190	CToken::Delete();
1191}
1192
1193void CCommentToken::Init(LPCTSTR name)
1194{
1195	CToken::Init();
1196	m_string = (char*)malloc(strlen(name) + 1);
1197//	ASSERT(m_string);
1198	strcpy(m_string, name);
1199}
1200
1201int CCommentToken::GetType()
1202{
1203	return TK_COMMENT;
1204}
1205
1206//
1207// CUserToken
1208//
1209
1210CUserToken::CUserToken()
1211{
1212}
1213
1214CUserToken::~CUserToken()
1215{
1216}
1217
1218CUserToken* CUserToken::Create(int value, LPCTSTR string)
1219{
1220	CUserToken* theToken = new CUserToken();
1221	theToken->Init(value, string);
1222	return theToken;
1223}
1224
1225void CUserToken::Delete()
1226{
1227	CToken::Delete();
1228}
1229
1230void CUserToken::Init(int value, LPCTSTR string)
1231{
1232	CToken::Init();
1233	m_value = value;
1234	m_string = (char*)malloc(strlen(string) + 1);
1235	strcpy(m_string, string);
1236}
1237
1238int CUserToken::GetType()
1239{
1240	return m_value;
1241}
1242
1243//
1244// CUndefinedToken
1245//
1246
1247CUndefinedToken::CUndefinedToken()
1248{
1249}
1250
1251CUndefinedToken::~CUndefinedToken()
1252{
1253}
1254
1255CUndefinedToken* CUndefinedToken::Create(LPCTSTR string)
1256{
1257	CUndefinedToken* theToken = new CUndefinedToken();
1258	theToken->Init(string);
1259	return theToken;
1260}
1261
1262void CUndefinedToken::Delete()
1263{
1264	CToken::Delete();
1265}
1266
1267void CUndefinedToken::Init(LPCTSTR string)
1268{
1269	CToken::Init();
1270	m_string = (char*)malloc(strlen(string) + 1);
1271	strcpy(m_string, string);
1272}
1273
1274int CUndefinedToken::GetType()
1275{
1276	return TK_UNDEFINED;
1277}
1278
1279//
1280// CTokenizerState
1281//
1282
1283CTokenizerState::CTokenizerState()
1284{
1285}
1286
1287CTokenizerState::~CTokenizerState()
1288{
1289}
1290
1291CTokenizerState* CTokenizerState::Create(bool skip)
1292{
1293	CTokenizerState* retval = new CTokenizerState();
1294	retval->Init(skip);
1295	return retval;
1296}
1297
1298void CTokenizerState::Init(bool skip)
1299{
1300	m_next = NULL;
1301	m_skip = skip;
1302	m_elseHit = false;
1303}
1304
1305void CTokenizerState::Delete()
1306{
1307	delete this;
1308}
1309
1310CTokenizerState* CTokenizerState::GetNext()
1311{
1312	return m_next;
1313}
1314
1315bool CTokenizerState::ProcessElse()
1316{
1317	if (!m_elseHit)
1318	{
1319		m_elseHit = true;
1320		m_skip = !m_skip;
1321	}
1322	return m_elseHit;
1323}
1324
1325void CTokenizerState::SetNext(CTokenizerState* next)
1326{
1327	m_next = next;
1328}
1329
1330bool CTokenizerState::Skipping()
1331{
1332	return m_skip;
1333}
1334
1335//
1336// CTokenizerHolderState
1337//
1338
1339CTokenizerHolderState::CTokenizerHolderState()
1340{
1341}
1342
1343CTokenizerHolderState::~CTokenizerHolderState()
1344{
1345}
1346
1347CTokenizerHolderState* CTokenizerHolderState::Create()
1348{
1349	CTokenizerHolderState* retval = new CTokenizerHolderState();
1350	retval->Init();
1351	return retval;
1352}
1353
1354void CTokenizerHolderState::Init()
1355{
1356	CTokenizerState::Init(true);
1357}
1358
1359void CTokenizerHolderState::Delete()
1360{
1361	delete this;
1362}
1363
1364bool CTokenizerHolderState::ProcessElse()
1365{
1366	if (!m_elseHit)
1367	{
1368		m_elseHit = true;
1369	}
1370	return m_elseHit;
1371}
1372
1373//
1374// CKeywordTable
1375//
1376
1377CKeywordTable::CKeywordTable(CTokenizer* tokenizer, keywordArray_t* keywords)
1378{
1379	m_tokenizer = tokenizer;
1380	m_holdKeywords = tokenizer->SetKeywords(keywords);
1381}
1382
1383CKeywordTable::~CKeywordTable()
1384{
1385	m_tokenizer->SetKeywords(m_holdKeywords);
1386}
1387
1388//
1389// CTokenizer
1390//
1391
1392CTokenizer::CTokenizer()
1393{
1394}
1395
1396CTokenizer::~CTokenizer()
1397{
1398}
1399
1400CTokenizer* CTokenizer::Create(UINT dwFlags)
1401{
1402	CTokenizer* theTokenizer = new CTokenizer();
1403	theTokenizer->Init(dwFlags);
1404	return theTokenizer;
1405}
1406
1407void CTokenizer::Delete()
1408{
1409	while (m_curParseStream != NULL)
1410	{
1411		CParseStream* curStream = m_curParseStream;
1412		m_curParseStream = curStream->GetNext();
1413		curStream->Delete();
1414	}
1415	if (m_symbolLookup != NULL)
1416	{
1417		m_symbolLookup->Delete();
1418		m_symbolLookup = NULL;
1419	}
1420	while (m_nextToken != NULL)
1421	{
1422		CToken* curToken = m_nextToken;
1423		m_nextToken = curToken->GetNext();
1424		curToken->Delete();
1425	}
1426	while (m_state != NULL)
1427	{
1428		Error(TKERR_UNMATCHED_DIRECTIVE);
1429		CTokenizerState* curState = m_state;
1430		m_state = curState->GetNext();
1431		curState->Delete();
1432	}
1433
1434/*	if (m_lastErrMsg != NULL)
1435	{
1436		free(m_lastErrMsg);
1437		m_lastErrMsg = NULL;
1438	}*/
1439	delete this;
1440}
1441
1442void CTokenizer::Error(int theError)
1443{
1444	char errString[128];
1445	char lookupstring[128];
1446	int i = 0;
1447	while ((errorMessages[i].m_tokenvalue != TKERR_USERERROR) && (errorMessages[i].m_tokenvalue != theError))
1448	{
1449		i++;
1450	}
1451	if ((errorMessages[i].m_tokenvalue == TKERR_USERERROR) && (m_errors != NULL))
1452	{
1453		i = 0;
1454		while ((m_errors[i].m_tokenvalue != TK_EOF) && (m_errors[i].m_tokenvalue != theError))
1455		{
1456			i++;
1457		}
1458		strcpy(lookupstring, m_errors[i].m_keyword);
1459	}
1460	else
1461	{
1462		strcpy(lookupstring, errorMessages[i].m_keyword);
1463	}
1464	sprintf(errString, "Error -- %d, %s", theError, lookupstring);
1465	Error(errString, theError);
1466}
1467
1468void CTokenizer::Error(int theError, LPCTSTR errString)
1469{
1470	char errstring[128];
1471	char lookupstring[128];
1472	int i = 0;
1473	while ((errorMessages[i].m_tokenvalue != TKERR_USERERROR) && (errorMessages[i].m_tokenvalue != theError))
1474	{
1475		i++;
1476	}
1477	if ((errorMessages[i].m_tokenvalue == TKERR_USERERROR) && (m_errors != NULL))
1478	{
1479		i = 0;
1480		while ((m_errors[i].m_tokenvalue != TK_EOF) && (m_errors[i].m_tokenvalue != theError))
1481		{
1482			i++;
1483		}
1484		strcpy(lookupstring, m_errors[i].m_keyword);
1485	}
1486	else
1487	{
1488		strcpy(lookupstring, errorMessages[i].m_keyword);
1489	}
1490	sprintf(errstring, "Error -- %d, %s - %s", theError, lookupstring, errString);
1491	Error(errstring, theError);
1492}
1493
1494void CTokenizer::Error(LPCTSTR errString, int theError)
1495{
1496	if (m_errorProc != NULL)
1497	{
1498		m_errorProc(errString);
1499	}
1500#ifdef USES_MODULES
1501	else 
1502	{
1503		ReportError(theError, errString);
1504	}
1505#endif
1506}
1507
1508bool CTokenizer::AddParseFile(LPCTSTR filename)
1509{
1510	CParseStream* newStream = CParseFile::Create(filename, this);
1511
1512	if ( newStream != NULL )
1513	{
1514		newStream->SetNext(m_curParseStream);
1515		m_curParseStream = newStream;
1516		return true;
1517	}
1518
1519	return false;
1520}
1521
1522void CTokenizer::AddParseStream(byte* data, long datasize)
1523{
1524	CParseStream* newStream = CParseMemory::Create(data, datasize);
1525	newStream->SetNext(m_curParseStream);
1526	m_curParseStream = newStream;
1527}
1528
1529long CTokenizer::GetRemainingSize()
1530{
1531	long retval = 0;
1532	CParseStream* curStream = m_curParseStream;
1533	while (curStream != NULL)
1534	{
1535		retval += curStream->GetRemainingSize();
1536		curStream = curStream->GetNext();
1537	}
1538	return retval;
1539}
1540
1541LPCTSTR CTokenizer::LookupToken(int tokenID, keywordArray_t* theTable)
1542{
1543	if (theTable == NULL)
1544	{
1545		theTable = m_keywords;
1546	}
1547	if (theTable == NULL)
1548	{
1549		return NULL;
1550	}
1551
1552	int i = 0;
1553	while (theTable[i].m_tokenvalue != TK_EOF)
1554	{
1555		if (theTable[i].m_tokenvalue == tokenID)
1556		{
1557			return theTable[i].m_keyword;
1558		}
1559		i++;
1560	}
1561	return NULL;
1562}
1563
1564void CTokenizer::PutBackToken(CToken* theToken, bool commented, LPCTSTR addedChars, bool bIgnoreThisTokenType)
1565{
1566	if (commented)
1567	{
1568		CParseToken* newStream = CParseToken::Create(theToken);
1569		newStream->SetNext(m_curParseStream);
1570		m_curParseStream = newStream;
1571
1572		if (addedChars != NULL)
1573		{
1574			CParsePutBack* spacer = CParsePutBack::Create(' ', 0, NULL);
1575			spacer->SetNext(m_curParseStream);
1576			m_curParseStream = spacer;
1577
1578			CParseBlock* newBlock = CParseBlock::Create((byte*)addedChars, strlen(addedChars));
1579			newBlock->SetNext(m_curParseStream);
1580			m_curParseStream = newBlock;
1581		}
1582
1583		char temp[] = "// * ";
1584		CParseBlock* newBlock = CParseBlock::Create((byte*)temp, strlen(temp));
1585		newBlock->SetNext(m_curParseStream);
1586		m_curParseStream = newBlock;
1587		return;
1588	}
1589
1590	switch(theToken->GetType())
1591	{
1592	case TK_INT:
1593	case TK_EOF:
1594	case TK_UNDEFINED:
1595	case TK_FLOAT:
1596	case TK_CHAR:
1597	case TK_STRING:
1598	case TK_EOL:
1599	case TK_COMMENT:
1600		if (!bIgnoreThisTokenType)
1601		{
1602			theToken->SetNext(m_nextToken);
1603			m_nextToken = theToken;
1604			break;
1605		}
1606	default:
1607		CParseToken* newStream = CParseToken::Create(theToken);
1608		newStream->SetNext(m_curParseStream);
1609		m_curParseStream = newStream;
1610		break;
1611	}
1612
1613	if (addedChars != NULL)
1614	{
1615		CParseBlock* newBlock = CParseBlock::Create((byte*)addedChars, strlen(addedChars));
1616		newBlock->SetNext(m_curParseStream);
1617		m_curParseStream = newBlock;
1618	}
1619}
1620
1621CToken* CTokenizer::GetToken(keywordArray_t* keywords, UINT onFlags, UINT offFlags)
1622{
1623	keywordArray_t* holdKeywords = SetKeywords(keywords);
1624	CToken* retval = GetToken(onFlags, offFlags);
1625	SetKeywords(holdKeywords);
1626	return retval;
1627}
1628
1629CToken* CTokenizer::GetToken(UINT onFlags, UINT offFlags)
1630{
1631	UINT holdFlags = m_flags;
1632
1633	m_flags |= onFlags;
1634	m_flags &= (~offFlags);
1635	CToken* theToken = NULL;
1636	while (theToken == NULL)
1637	{
1638		theToken = FetchToken();
1639		if (theToken == NULL)
1640		{
1641			continue;
1642		}
1643		if (theToken->GetType() == TK_EOF)
1644		{
1645			break;
1646		}
1647		if (m_state != NULL)
1648		{
1649			if (m_state->Skipping())
1650			{
1651				theToken->Delete();
1652				theToken = NULL;
1653			}
1654		}
1655	}
1656
1657	m_flags = holdFlags;
1658	return theToken;
1659}
1660
1661CToken* CTokenizer::GetToEndOfLine(int tokenType)
1662{
1663	// update, if you just want the whole line returned as a string, then allow a much bigger size than
1664	//	the default string size of only 128 chars...
1665	//
1666	if (tokenType == TK_STRING)
1667	{		
1668		#define iRETURN_STRING_SIZE 2048
1669		char theString[iRETURN_STRING_SIZE];
1670		theString[0] = ' ';
1671
1672		for (int i = 1; i < iRETURN_STRING_SIZE; i++)
1673		{
1674			if (NextChar((byte&)theString[i]))
1675			{
1676				if (theString[i] != '\n')
1677				{
1678					continue;
1679				}
1680				PutBackChar(theString[i]);
1681			}
1682			theString[i] = '\0';
1683
1684			return CStringToken::Create(theString);			
1685		}
1686
1687		// line would maks a string too big to fit in buffer...
1688		//			
1689		Error(TKERR_STRINGLENGTHEXCEEDED);
1690	}
1691	else
1692	{		
1693		char theString[MAX_IDENTIFIER_LENGTH];
1694		theString[0] = ' ';
1695		while (theString[0] == ' ')
1696		{
1697			if (!NextChar((byte&)theString[0]))
1698			{
1699				return NULL;
1700			}
1701		}
1702		for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
1703		{
1704			if (NextChar((byte&)theString[i]))
1705			{
1706				if (theString[i] != '\n')
1707				{
1708					continue;
1709				}
1710				PutBackChar(theString[i]);
1711			}
1712			theString[i] = '\0';
1713			switch(tokenType)
1714			{
1715			case TK_COMMENT:
1716				return CCommentToken::Create(theString);
1717			case TK_IDENTIFIER:
1718			default:
1719				return CIdentifierToken::Create(theString);
1720			}
1721		}
1722		Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
1723	}	
1724	return NULL;
1725}
1726
1727void CTokenizer::SkipToLineEnd()
1728{
1729	byte theByte;
1730	while(NextChar(theByte))
1731	{
1732		if (theByte == '\n')
1733		{
1734			break;
1735		}
1736	}
1737}
1738
1739CToken* CTokenizer::FetchToken()
1740{
1741	if (m_nextToken != NULL)
1742	{
1743		CToken* curToken = m_nextToken;
1744		m_nextToken = curToken->GetNext();
1745		curToken->SetNext(NULL);
1746		return curToken;
1747	}
1748	byte theByte;
1749	CToken* theToken = NULL;
1750
1751	while (true)
1752	{
1753		if (!NextChar(theByte))
1754		{
1755			return CToken::Create();
1756		}
1757		if (theByte <= ' ')
1758		{
1759			if ((theByte == '\n') && ((TKF_USES_EOL & m_flags) != 0))
1760			{
1761				return CUserToken::Create(TK_EOL, "-EOLN-");
1762			}
1763			continue;
1764		}
1765		switch(theByte)
1766		{
1767		case '#':
1768			if ((m_flags & TKF_IGNOREDIRECTIVES) == 0)
1769			{
1770				theToken = HandleDirective();
1771			}
1772			else
1773			{
1774				if ((m_flags & TKF_NODIRECTIVES) != 0)
1775				{
1776					return HandleSymbol('#');
1777				}
1778				SkipToLineEnd();
1779			}
1780			break;
1781		case '/':
1782			theToken = HandleSlash();
1783			break;
1784		case '"':
1785			theToken = HandleString();
1786			break;
1787		case '\'':
1788			theToken = HandleQuote();
1789			break;
1790		default:
1791			if (((theByte >= 'a') && (theByte <= 'z'))
1792				|| ((theByte >= 'A') && (theByte <= 'Z'))
1793				|| ((theByte == '_') && ((m_flags & TKF_NOUNDERSCOREINIDENTIFIER) == 0)))
1794			{
1795				theToken = HandleIdentifier(theByte);
1796			}
1797			else if (((m_flags & TKF_NUMERICIDENTIFIERSTART) != 0) && (theByte >= '0') && (theByte <= '9'))
1798			{
1799				theToken = HandleIdentifier(theByte);
1800			}
1801			else if (((theByte >= '0') && (theByte <= '9')) || (theByte == '-'))
1802			{
1803				theToken = HandleNumeric(theByte);
1804			}
1805			else if (theByte == '.')
1806			{
1807				theToken = HandleDecimal();
1808			}
1809			else if (theByte <= ' ')
1810			{
1811				break;
1812			}
1813			else
1814			{
1815				theToken = HandleSymbol(theByte);
1816			}
1817		}
1818		if (theToken != NULL)
1819		{
1820			return theToken;
1821		}
1822	}
1823}
1824
1825bool CTokenizer::NextChar(byte& theByte)
1826{
1827	while (m_curParseStream != NULL)
1828	{
1829		if (m_curParseStream->NextChar(theByte))
1830		{
1831			return true;
1832		}
1833		CParseStream* curParseStream = m_curParseStream;
1834		m_curParseStream = curParseStream->GetNext();
1835		curParseStream->Delete();
1836	}
1837	return false;
1838}
1839
1840bool CTokenizer::RequireToken(int tokenType)
1841{
1842	CToken* theToken = GetToken();
1843	bool retValue = theToken->GetType() == tokenType;
1844	theToken->Delete();
1845	return retValue;
1846}
1847
1848void CTokenizer::ScanUntilToken(int tokenType)
1849{
1850	CToken* curToken;
1851	int tokenValue = TK_UNDEFINED;
1852	while (tokenValue != tokenType)
1853	{
1854		curToken = GetToken();
1855		tokenValue = curToken->GetType();
1856		if (tokenValue == TK_EOF)
1857		{
1858			PutBackToken(curToken);
1859			break;
1860		}
1861		if (tokenValue == tokenType)
1862		{
1863			PutBackToken(curToken);
1864		}
1865		else
1866		{
1867			curToken->Delete();
1868		}
1869	}
1870}
1871
1872void CTokenizer::Init(UINT dwFlags)
1873{
1874	m_symbolLookup = NULL;
1875	m_nextToken = NULL;
1876	m_curParseStream = NULL;
1877	m_keywords = NULL;
1878	m_symbols = NULL;
1879	m_errors = NULL;
1880	m_state = NULL;
1881	m_flags = dwFlags;
1882	m_errorProc = NULL;
1883}
1884
1885void CTokenizer::SetErrorProc(LPTokenizerErrorProc errorProc)
1886{
1887	m_errorProc = errorProc;
1888}
1889
1890void CTokenizer::SetAdditionalErrors(keywordArray_t* theErrors)
1891{
1892	m_errors = theErrors;
1893}
1894
1895keywordArray_t* CTokenizer::SetKeywords(keywordArray_t* theKeywords)
1896{
1897	keywordArray_t* retval = m_keywords;
1898	m_keywords = theKeywords;
1899	return retval;
1900}
1901
1902void CTokenizer::SetSymbols(keywordArray_t* theSymbols)
1903{
1904	m_symbols = theSymbols;
1905	if (m_symbolLookup != NULL)
1906	{
1907		m_symbolLookup->Delete();
1908		m_symbolLookup = NULL;
1909	}
1910	int i = 0;
1911	if (theSymbols == NULL)
1912	{
1913		return;
1914	}
1915	while(theSymbols[i].m_tokenvalue != TK_EOF)
1916	{
1917		InsertSymbol(theSymbols[i].m_keyword, theSymbols[i].m_tokenvalue);
1918		i++;
1919	}
1920}
1921
1922void CTokenizer::InsertSymbol(LPCTSTR theSymbol, int theValue)
1923{
1924	CSymbolLookup** curHead = &m_symbolLookup;
1925	CSymbolLookup* curParent = NULL;
1926	CSymbolLookup* curLookup = NULL;
1927
1928	for (UINT i = 0; i < strlen(theSymbol); i++)
1929	{
1930		bool found = false;
1931		curLookup = *curHead;
1932		while (curLookup != NULL)
1933		{
1934			if (curLookup->GetByte() == theSymbol[i])
1935			{
1936				found = true;
1937				break;
1938			}
1939			curLookup = curLookup->GetNext();
1940		}
1941		if (!found)
1942		{
1943			curLookup = CSymbolLookup::Create(theSymbol[i]);
1944			curLookup->SetParent(curParent);
1945			curLookup->SetNext(*curHead);
1946			*curHead = curLookup;
1947		}
1948		curHead = curLookup->GetChildAddress();
1949		curParent = curLookup;
1950	}
1951	if (curLookup->GetValue() != -1)
1952	{
1953		Error(TKERR_DUPLICATESYMBOL);
1954	}
1955	curLookup->SetValue(theValue);
1956}
1957
1958CToken* CTokenizer::HandleString()
1959{
1960	char theString[MAX_STRING_LENGTH];
1961	for (int i = 0; i < MAX_STRING_LENGTH; i++)
1962	{
1963		if (!NextChar((byte&)theString[i]))
1964		{
1965			return NULL;
1966		}
1967		if (theString[i] == '"')
1968		{
1969			theString[i] = '\0';
1970			return CStringToken::Create(theString);
1971		}
1972		if (theString[i] == '\\')
1973		{
1974			theString[i] = Escapement();
1975		}
1976	}
1977	Error(TKERR_STRINGLENGTHEXCEEDED);
1978	return NULL;
1979}
1980
1981void CTokenizer::GetCurFilename(char** filename)
1982{
1983	if (m_curParseStream == NULL)
1984	{
1985		*filename = (char*)malloc(1);
1986		*filename[0] = '\0';
1987		return;
1988	}
1989	m_curParseStream->GetCurFilename(filename);
1990}
1991
1992int CTokenizer::GetCurLine()
1993{
1994	if (m_curParseStream == NULL)
1995	{
1996		return 0;
1997	}
1998	return m_curParseStream->GetCurLine();
1999}
2000
2001void CTokenizer::PutBackChar(byte theByte, int curLine, LPCTSTR filename)
2002{
2003	CParseStream* newStream;
2004	if (filename == NULL)
2005	{
2006		curLine = m_curParseStream->GetCurLine();
2007		char* theFile = NULL;
2008		m_curParseStream->GetCurFilename(&theFile);
2009		newStream = CParsePutBack::Create(theByte, curLine, theFile);
2010		if (theFile != NULL)
2011		{
2012			free(theFile);
2013		}
2014	}
2015	else
2016	{
2017		newStream = CParsePutBack::Create(theByte, curLine, filename);
2018	}
2019	newStream->SetNext(m_curParseStream);
2020	m_curParseStream = newStream;
2021}
2022
2023byte CTokenizer::Escapement()
2024{
2025	byte theByte;
2026	if (NextChar(theByte))
2027	{
2028		switch(theByte)
2029		{
2030		case 'n':
2031			return '\n';
2032		case '\\':
2033			return '\\';
2034		case '\'':
2035			return '\'';
2036		case '?':
2037			return '\?';
2038		case 'a':
2039			return '\a';
2040		case 'b':
2041			return '\b';
2042		case 'f':
2043			return '\f';
2044		case 'r':
2045			return '\r';
2046		case 't':
2047			return '\t';
2048		case 'v':
2049			return '\v';
2050		case '0':
2051			return '\0';
2052		// support for octal or hex sequences?? \000 or \xhhh
2053		default:
2054			PutBackChar(theByte);
2055		}
2056	}
2057	return '\\';
2058}
2059
2060bool CTokenizer::AddDefineSymbol(CDirectiveSymbol* definesymbol)
2061{
2062	CParseStream* curStream = m_curParseStream;
2063	while(curStream != NULL)
2064	{
2065		if (curStream->IsThisDefinition(definesymbol))
2066		{
2067			return false;
2068		}
2069		curStream = curStream->GetNext();
2070	}
2071	CParseStream* newStream = CParseDefine::Create(definesymbol);
2072	newStream->SetNext(m_curParseStream);
2073	m_curParseStream = newStream;
2074	return true;
2075}
2076
2077CToken* CTokenizer::TokenFromName(LPCTSTR name)
2078{
2079	CDirectiveSymbol* defineSymbol = (CDirectiveSymbol*)m_defines.FindSymbol(name);
2080	if (defineSymbol != NULL)
2081	{
2082		if (AddDefineSymbol(defineSymbol))
2083		{
2084			return FetchToken();
2085		}
2086	}
2087	if ((m_keywords != NULL) && ((m_flags & TKF_IGNOREKEYWORDS) == 0))
2088	{
2089		int i = 0;
2090		if ((m_flags & TKF_NOCASEKEYWORDS) == 0)
2091		{
2092			while (m_keywords[i].m_tokenvalue != TK_EOF)
2093			{
2094				if (strcmp(m_keywords[i].m_keyword, name) == 0)
2095				{
2096					return CUserToken::Create(m_keywords[i].m_tokenvalue, name);
2097				}
2098				i++;
2099			}
2100		}
2101		else
2102		{
2103			while (m_keywords[i].m_tokenvalue != TK_EOF)
2104			{
2105				if (stricmp(m_keywords[i].m_keyword, name) == 0)
2106				{
2107					return CUserToken::Create(m_keywords[i].m_tokenvalue, name);
2108				}
2109				i++;
2110			}
2111		}
2112	}
2113	return CIdentifierToken::Create(name);
2114}
2115
2116int CTokenizer::DirectiveFromName(LPCTSTR name)
2117{
2118	if (directiveKeywords != NULL)
2119	{
2120		int i = 0;
2121		while (directiveKeywords[i].m_tokenvalue != TK_EOF)
2122		{
2123			if (strcmp(directiveKeywords[i].m_keyword, name) == 0)
2124			{
2125				return directiveKeywords[i].m_tokenvalue;
2126			}
2127			i++;
2128		}
2129	}
2130	return -1;
2131}
2132
2133CToken* CTokenizer::HandleIdentifier(byte theByte)
2134{
2135	char theString[MAX_IDENTIFIER_LENGTH];
2136	theString[0] = theByte;
2137	for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
2138	{
2139		if (NextChar((byte&)theString[i]))
2140		{
2141			if (((theString[i] != '_') || ((m_flags & TKF_NOUNDERSCOREINIDENTIFIER) == 0)) &&
2142				((theString[i] != '-') || ((m_flags & TKF_NODASHINIDENTIFIER) == 0)))
2143			{
2144				if (((theString[i] >= 'A') && (theString[i] <= 'Z'))
2145				 || ((theString[i] >= 'a') && (theString[i] <= 'z'))
2146				 || ((theString[i] >= '0') && (theString[i] <= '9'))
2147				 || (theString[i] == '_') || (theString[i] == '-'))
2148				{
2149					continue;
2150				}
2151			}
2152			PutBackChar(theString[i]);
2153		}
2154		theString[i] = '\0';
2155		return TokenFromName(theString);
2156	}
2157	Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
2158	return NULL;
2159}
2160
2161CToken* CTokenizer::HandleSlash()
2162{
2163	byte theByte;
2164	if (!NextChar(theByte))
2165	{
2166		return NULL;
2167	}
2168	if (theByte == '/')
2169	{
2170		if (m_flags & TKF_COMMENTTOKENS)
2171		{
2172			return GetToEndOfLine(TK_COMMENT);
2173		}
2174		SkipToLineEnd();
2175		return NULL;
2176	}
2177	if (theByte == '*')
2178	{
2179		if (m_flags & TKF_COMMENTTOKENS)
2180		{
2181			char theString[MAX_IDENTIFIER_LENGTH + 1];
2182			theString[0] = ' ';
2183			while (theString[0] == ' ')
2184			{
2185				if (!NextChar((byte&)theString[0]))
2186				{
2187					return NULL;
2188				}
2189			}
2190			for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
2191			{
2192				if (NextChar((byte&)theString[i]))
2193				{
2194					if (theString[i] != '*')
2195					{
2196						continue;
2197					}
2198					i++;
2199					if (NextChar((byte&)theString[i]))
2200					{
2201						if (theString[i] == '/')
2202						{
2203							i--;
2204							theString[i] = '\0';
2205							return CCommentToken::Create(theString);
2206						}
2207					}
2208				}
2209			}
2210			Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
2211			return NULL;
2212		}
2213		while(NextChar(theByte))
2214		{
2215			while(theByte == '*')
2216			{
2217				if (!NextChar(theByte))
2218				{
2219					break;
2220				}
2221				if (theByte == '/')
2222				{
2223					return NULL;
2224				}
2225			}
2226		}
2227		return NULL;
2228	}
2229	PutBackChar(theByte);
2230	return HandleSymbol('/');
2231}
2232
2233CToken* CTokenizer::HandleNumeric(byte theByte)
2234{
2235	bool thesign = theByte == '-';
2236	if (thesign)
2237	{
2238		if (!NextChar(theByte))
2239		{
2240			return HandleSymbol('-');
2241		}
2242		if (theByte == '.')
2243		{
2244			return HandleDecimal(thesign);
2245		}
2246		if ((theByte < '0') || (theByte > '9'))
2247		{
2248			PutBackChar(theByte);
2249			return HandleSymbol('-');
2250		}
2251	}
2252	if (theByte == '0')
2253	{
2254		return HandleOctal(thesign);
2255	}
2256	long value = 0;
2257	bool digithit = false;
2258	while((theByte >= '0') && (theByte <= '9'))
2259	{
2260		value = (value * 10) + (theByte - '0');
2261		if (!NextChar(theByte))
2262		{
2263			if (thesign)
2264			{
2265				value = -value;
2266			}
2267			return CIntToken::Create(value);
2268		}
2269		digithit = true;
2270	}
2271	if (theByte == '.')
2272	{
2273		if (digithit)
2274		{
2275			return HandleFloat(thesign, value);
2276		}
2277		if (thesign)
2278		{
2279			PutBackChar(theByte);
2280			theByte = '-';
2281		}
2282		return HandleSymbol(theByte);
2283	}
2284	PutBackChar(theByte);
2285	if (thesign)
2286	{
2287		value = -value;
2288	}
2289	return CIntToken::Create(value);
2290}
2291
2292CToken* CTokenizer::HandleOctal(bool thesign)
2293{
2294	byte theByte;
2295	int value = 0;
2296
2297	if (!NextChar(theByte))
2298	{
2299		return CIntToken::Create(value);
2300	}
2301	if (theByte == '.')
2302	{
2303		return HandleDecimal(thesign);
2304	}
2305	if ((theByte == 'x') || (theByte == 'X'))
2306	{
2307		return HandleHex(thesign);
2308	}
2309	while(true)
2310	{
2311		if((theByte >= '0') && (theByte <='7'))
2312		{
2313			value = (value * 8) + (theByte - '0');
2314		}
2315		else
2316		{
2317			PutBackChar(theByte);
2318			if (thesign)
2319			{
2320				value = -value;
2321			}
2322			return CIntToken::Create(value);
2323		}
2324		if (!NextChar(theByte))
2325		{
2326			if (thesign)
2327			{
2328				value = -value;
2329			}
2330			return CIntToken::Create(value);
2331		}
2332	}
2333}
2334
2335CToken* CTokenizer::HandleHex(bool thesign)
2336{
2337	int value = 0;
2338
2339	while (true)
2340	{
2341		byte theByte;
2342		if (!NextChar(theByte))
2343		{
2344			if (thesign)
2345			{
2346				value = -value;
2347			}
2348			return CIntToken::Create(value);
2349		}
2350		switch (theByte)
2351		{
2352		case '0':
2353		case '1':
2354		case '2':
2355		case '3':
2356		case '4':
2357		case '5':
2358		case '6':
2359		case '7':
2360		case '8':
2361		case '9':
2362			theByte = theByte - '0';
2363			break;
2364		case 'A':
2365		case 'B':
2366		case 'C':
2367		case 'D':
2368		case 'E':
2369		case 'F':
2370			theByte = theByte - 'A' + 10;
2371			break;
2372		case 'a':
2373		case 'b':
2374		case 'c':
2375		case 'd':
2376		case 'e':
2377		case 'f':
2378			theByte = theByte - 'a' + 10;
2379			break;
2380		default:
2381			PutBackChar(theByte);
2382			if (thesign)
2383			{
2384				value = -value;
2385			}
2386			return CIntToken::Create(value);
2387		}
2388		value = (value * 16) + theByte;
2389	}
2390}
2391
2392CToken* CTokenizer::HandleDecimal(bool thesign)
2393{
2394	byte theByte;
2395	if (!NextChar(theByte))
2396	{
2397		if (thesign)
2398		{
2399			PutBackChar('.');
2400			return HandleSymbol('-');
2401		}
2402		HandleSymbol('.');
2403	}
2404	PutBackChar(theByte);
2405	if ((theByte <= '9') && (theByte >= '0'))
2406	{
2407		return HandleFloat(thesign);
2408	}
2409	if (thesign)
2410	{
2411		PutBackChar('.');
2412		theByte = '-';
2413	}
2414	else
2415	{
2416		theByte = '.';
2417	}
2418	return HandleSymbol(theByte);
2419}
2420
2421CToken* CTokenizer::HandleFloat(bool thesign, long value)
2422{
2423	float lower = 1.0;
2424	float newValue = (float)value;
2425	byte theByte;
2426	while(NextChar(theByte))
2427	{
2428		if ((theByte >= '0') && (theByte <= '9'))
2429		{
2430			lower = lower / 10;
2431			newValue = newValue + ((theByte - '0') * lower);
2432			continue;
2433		}
2434		PutBackChar(theByte);
2435		break;
2436	}
2437	if (thesign)
2438	{
2439		newValue = -newValue;
2440	}
2441	return CFloatToken::Create(newValue);
2442}
2443
2444CToken* CTokenizer::HandleQuote()
2445{
2446	byte theByte;
2447	if (!NextChar(theByte))
2448	{
2449		Error(TKERR_EXPECTED_CHAR);
2450		return NULL;
2451	}
2452	if (theByte == '\\')
2453	{
2454		theByte = Escapement();
2455	}
2456	byte dummy;
2457	if (!NextChar(dummy))
2458	{
2459		Error(TKERR_EXPECTED_CHAR);
2460		return NULL;
2461	}
2462	if (dummy != '\'')
2463	{
2464		PutBackChar(dummy);
2465		PutBackChar(theByte);
2466		Error(TKERR_EXPECTED_CHAR);
2467		return NULL;
2468	}
2469	return CCharToken::Create(theByte);
2470}
2471
2472void CTokenizer::SetFlags(UINT flags)
2473{
2474	m_flags = flags;
2475}
2476
2477UINT CTokenizer::GetFlags()
2478{
2479	return m_flags;
2480}
2481
2482CToken* CTokenizer::HandleSymbol(byte theByte)
2483{
2484	char symbolString[128];
2485	int curStrLen = 0;
2486	symbolString[0] = '\0';
2487	bool consumed = false;
2488
2489	CSymbolLookup* curLookup;
2490	if ((m_flags & TKF_RAWSYMBOLSONLY) == 0)
2491	{
2492		curLookup = m_symbolLookup;
2493	}
2494	else
2495	{
2496		curLookup = NULL;
2497	}
2498	CSymbolLookup* lastLookup = NULL;
2499	while(curLookup != NULL)
2500	{
2501		if (curLookup->GetByte() == theByte)
2502		{
2503			symbolString[curStrLen++] = theByte;
2504			symbolString[curStrLen] = '\0';
2505			lastLookup = curLookup;
2506			consumed = true;
2507			if (curLookup->GetChild() == NULL)
2508			{
2509				break;
2510			}
2511			if (!NextChar(theByte))
2512			{
2513				PutBackToken(CToken::Create());
2514				break;
2515			}
2516			consumed = false;
2517			curLookup = curLookup->GetChild();
2518			continue;
2519		}
2520		curLookup = curLookup->GetNext();
2521	}
2522	if ((!consumed) && (lastLookup != NULL))
2523	{
2524		PutBackChar(theByte);
2525	}
2526	while ((lastLookup != NULL) && (lastLookup->GetValue() == -1))
2527	{
2528		curStrLen--;
2529		symbolString[curStrLen] = '\0';
2530	//	symbolString = symbolString.Left(symbolString.GetLength() - 1);
2531		if (lastLookup->GetParent() == NULL)
2532		{
2533			if ((m_flags & TKF_WANTUNDEFINED) == 0)
2534			{
2535				Error(TKERR_UNRECOGNIZEDSYMBOL);
2536			}
2537		}
2538		else
2539		{
2540			PutBackChar(lastLookup->GetByte());
2541		}
2542		lastLookup = lastLookup->GetParent();
2543	}
2544	if (lastLookup == NULL)
2545	{
2546		if ((m_flags & TKF_WANTUNDEFINED) == 0)
2547		{
2548			return NULL;
2549		}
2550		curStrLen = 0;
2551		symbolString[curStrLen++] = char(theByte);
2552		symbolString[curStrLen] = '\0';
2553		if ((m_flags & TKF_WIDEUNDEFINEDSYMBOLS) != 0)
2554		{
2555			while (true)
2556			{
2557				if (!NextChar(theByte))
2558				{
2559					PutBackToken(CToken::Create());
2560					break;
2561				}
2562				if (theByte == ' ')
2563				{
2564					break;
2565				}
2566				if (((theByte >= 'a') && (theByte <= 'z')) || ((theByte >= 'A') && (theByte <= 'Z')) ||
2567				((theByte >= '0') && (theByte <= '9')))
2568				{
2569					PutBackChar(theByte);
2570					break;
2571				}
2572				if (theByte < ' ')
2573				{
2574					if ((theByte == '\n') && ((TKF_USES_EOL & m_flags) != 0))
2575					{
2576						PutBackToken(CUserToken::Create(TK_EOL, "-EOLN-"));
2577						break;
2578					}
2579					continue;
2580				}
2581				symbolString[curStrLen++] = theByte;
2582				symbolString[curStrLen] = '\0';
2583			}
2584		}
2585		return CUndefinedToken::Create(symbolString);
2586	}
2587	return CUserToken::Create(lastLookup->GetValue(), symbolString);
2588}
2589
2590CToken* CTokenizer::HandleDirective()
2591{
2592	int tokenValue = 0;
2593	CToken* theToken = FetchToken();
2594	if (theToken->GetType() == TK_EOF)
2595	{
2596		return theToken;
2597	}
2598	if (theToken->GetType() != TK_IDENTIFIER)
2599	{
2600		Error(TKERR_INVALID_DIRECTIVE);
2601		theToken->Delete();
2602		SkipToLineEnd();
2603		return NULL;
2604	}
2605
2606	CDirectiveSymbol* curSymbol;
2607	CTokenizerState* state;
2608	int theDirective = DirectiveFromName(theToken->GetStringValue());
2609	theToken->Delete();
2610	byte theByte;
2611	switch(theDirective)
2612	{
2613	case DIR_INCLUDE:
2614		if ((m_state != NULL) && (m_state->Skipping()))
2615		{
2616			break;
2617		}
2618		theToken = GetToken();
2619		if (theToken->GetType() != TK_STRING)
2620		{
2621			Error(TKERR_INCLUDE_FILE_NOTFOUND);
2622			theToken->Delete();
2623			SkipToLineEnd();
2624			break;
2625		}
2626		AddParseFile(theToken->GetStringValue());
2627		theToken->Delete();
2628		break;
2629	case DIR_IFDEF:
2630		if ((m_state != NULL) && (m_state->Skipping()))
2631		{
2632			state = CTokenizerHolderState::Create();
2633			state->SetNext(m_state);
2634			m_state = state;
2635			break;
2636		}
2637		theToken = GetToken();
2638		if (theToken->GetType() != TK_IDENTIFIER)
2639		{
2640			Error(TKERR_EXPECTED_IDENTIFIER);
2641			theToken->Delete();
2642			SkipToLineEnd();
2643			break;
2644		}
2645		state = CTokenizerState::Create(m_defines.FindSymbol(theToken->GetStringValue()) == NULL);
2646		theToken->Delete();
2647		state->SetNext(m_state);
2648		m_state = state;
2649		break;
2650	case DIR_IFNDEF:
2651		if ((m_state != NULL) && (m_state->Skipping()))
2652		{
2653			state = CTokenizerHolderState::Create();
2654			state->SetNext(m_state);
2655			m_state = state;
2656			break;
2657		}
2658		theToken = GetToken();
2659		if (theToken->GetType() != TK_IDENTIFIER)
2660		{
2661			Error(TKERR_EXPECTED_IDENTIFIER);
2662			theToken->Delete();
2663			SkipToLineEnd();
2664			break;
2665		}
2666		state = CTokenizerState::Create(m_defines.FindSymbol(theToken->GetStringValue()) != NULL);
2667		theToken->Delete();
2668		state->SetNext(m_state);
2669		m_state = state;
2670		break;
2671	case DIR_ENDIF:
2672		if (m_state == NULL)
2673		{
2674			Error(TKERR_UNMATCHED_DIRECTIVE);
2675			break;
2676		}
2677		state = m_state;
2678		m_state = state->GetNext();
2679		state->Delete();
2680		break;
2681	case DIR_ELSE:
2682		if (m_state == NULL)
2683		{
2684			Error(TKERR_UNMATCHED_DIRECTIVE);
2685			break;
2686		}
2687		if (!m_state->ProcessElse())
2688		{
2689			Error(TKERR_UNMATCHED_DIRECTIVE);
2690			break;
2691		}
2692		break;
2693	case DIR_DEFINE:
2694		if ((m_state != NULL) && (m_state->Skipping()))
2695		{
2696			break;
2697		}
2698		theToken = GetToken();
2699		if (theToken->GetType() != TK_IDENTIFIER)
2700		{
2701			Error(TKERR_EXPECTED_IDENTIFIER);
2702			theToken->Delete();
2703			SkipToLineEnd();
2704			break;
2705		}
2706		// blind add - should check first for value changes
2707		{
2708			curSymbol = CDirectiveSymbol::Create(theToken->GetStringValue());
2709			char temp[128];
2710			int tempsize = 0;
2711			while(NextChar(theByte))
2712			{
2713				if (theByte == '\n')
2714				{
2715					break;
2716				}
2717				temp[tempsize++] = char(theByte);
2718			}
2719			temp[tempsize] = '\0';
2720			curSymbol->SetValue(temp);
2721			if (!m_defines.AddSymbol(curSymbol))
2722			{
2723				curSymbol->Delete();
2724			}
2725		}
2726		break;
2727	case DIR_UNDEFINE:
2728		if ((m_state != NULL) && (m_state->Skipping()))
2729		{
2730			break;
2731		}
2732		theToken = GetToken();
2733		if (theToken->GetType() != TK_IDENTIFIER)
2734		{
2735			Error(TKERR_EXPECTED_IDENTIFIER);
2736			theToken->Delete();
2737			SkipToLineEnd();
2738			break;
2739		}
2740		m_defines.RemoveSymbol(theToken->GetStringValue());
2741		break;
2742	default:
2743		Error(TKERR_INVALID_DIRECTIVE);
2744		SkipToLineEnd();
2745		break;
2746	}
2747	return NULL;
2748}
2749
2750COLORREF CTokenizer::ParseRGB()
2751{
2752	CToken* theToken = GetToken();
2753	if (theToken->GetType() != TK_INT)
2754	{
2755		Error(TKERR_EXPECTED_INTEGER);
2756		theToken->Delete();
2757		return RGB(0, 0, 0);
2758	}
2759	int red = theToken->GetIntValue();
2760	theToken->Delete();
2761	theToken = GetToken();
2762	if (theToken->GetType() != TK_INT)
2763	{
2764		Error(TKERR_EXPECTED_INTEGER);
2765		theToken->Delete();
2766		return RGB(0, 0, 0);
2767	}
2768	int green = theToken->GetIntValue();
2769	theToken->Delete();
2770	theToken = GetToken();
2771	if (theToken->GetType() != TK_INT)
2772	{
2773		Error(TKERR_EXPECTED_INTEGER);
2774		theToken->Delete();
2775		return RGB(0, 0, 0);
2776	}
2777	int blue = theToken->GetIntValue();
2778	theToken->Delete();
2779	return RGB(red, green, blue);
2780}
2781
2782//
2783// CSymbolLookup
2784//
2785
2786CSymbolLookup::CSymbolLookup()
2787{
2788}
2789
2790CSymbolLookup::~CSymbolLookup()
2791{
2792}
2793
2794CSymbolLookup* CSymbolLookup::Create(byte theByte)
2795{
2796	CSymbolLookup* curLookup = new CSymbolLookup();
2797	curLookup->Init(theByte);
2798	return curLookup;
2799}
2800
2801void CSymbolLookup::Delete()
2802{
2803	if (m_sibling != NULL)
2804	{
2805		m_sibling->Delete();
2806		m_sibling = NULL;
2807	}
2808	if (m_child != NULL)
2809	{
2810		m_child->Delete();
2811		m_child = NULL;
2812	}
2813	delete this;
2814}
2815
2816void CSymbolLookup::Init(byte theByte)
2817{
2818	m_parent = NULL;
2819	m_child = NULL;
2820	m_sibling = NULL;
2821	m_value = -1;
2822	m_byte = theByte;
2823}
2824
2825CSymbolLookup* CSymbolLookup::GetNext()
2826{
2827	return m_sibling;
2828}
2829
2830void CSymbolLookup::SetNext(CSymbolLookup* next)
2831{
2832	m_sibling = next;
2833}
2834
2835void CSymbolLookup::SetParent(CSymbolLookup* parent)
2836{
2837	m_parent = parent;
2838}
2839
2840void CSymbolLookup::SetValue(int value)
2841{
2842	m_value = value;
2843}
2844
2845int CSymbolLookup::GetValue()
2846{
2847	return m_value;
2848}
2849
2850byte CSymbolLookup::GetByte()
2851{
2852	return m_byte;
2853}
2854
2855CSymbolLookup** CSymbolLookup::GetChildAddress()
2856{
2857	return &m_child;
2858}
2859
2860CSymbolLookup* CSymbolLookup::GetChild()
2861{
2862	return m_child;
2863}
2864
2865CSymbolLookup* CSymbolLookup::GetParent()
2866{
2867	return m_parent;
2868}