PageRenderTime 63ms CodeModel.GetById 19ms app.highlight 39ms RepoModel.GetById 1ms app.codeStats 0ms

/indra/llui/llkeywords.cpp

https://bitbucket.org/lindenlab/viewer-beta/
C++ | 667 lines | 516 code | 80 blank | 71 comment | 125 complexity | d992b4d8b4f4caada6502222898d30e4 MD5 | raw file
  1/** 
  2 * @file llkeywords.cpp
  3 * @brief Keyword list for LSL
  4 *
  5 * $LicenseInfo:firstyear=2000&license=viewerlgpl$
  6 * Second Life Viewer Source Code
  7 * Copyright (C) 2010, Linden Research, Inc.
  8 * 
  9 * This library is free software; you can redistribute it and/or
 10 * modify it under the terms of the GNU Lesser General Public
 11 * License as published by the Free Software Foundation;
 12 * version 2.1 of the License only.
 13 * 
 14 * This library is distributed in the hope that it will be useful,
 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 17 * Lesser General Public License for more details.
 18 * 
 19 * You should have received a copy of the GNU Lesser General Public
 20 * License along with this library; if not, write to the Free Software
 21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 22 * 
 23 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
 24 * $/LicenseInfo$
 25 */
 26
 27#include "linden_common.h"
 28
 29#include <iostream>
 30#include <fstream>
 31
 32#include "llkeywords.h"
 33#include "lltexteditor.h"
 34#include "llstl.h"
 35#include <boost/tokenizer.hpp>
 36
 37const U32 KEYWORD_FILE_CURRENT_VERSION = 2;
 38
 39inline BOOL LLKeywordToken::isHead(const llwchar* s) const
 40{
 41	// strncmp is much faster than string compare
 42	BOOL res = TRUE;
 43	const llwchar* t = mToken.c_str();
 44	S32 len = mToken.size();
 45	for (S32 i=0; i<len; i++)
 46	{
 47		if (s[i] != t[i])
 48		{
 49			res = FALSE;
 50			break;
 51		}
 52	}
 53	return res;
 54}
 55
 56LLKeywords::LLKeywords() : mLoaded(FALSE)
 57{
 58}
 59
 60inline BOOL LLKeywordToken::isTail(const llwchar* s) const
 61{
 62	BOOL res = TRUE;
 63	const llwchar* t = mDelimiter.c_str();
 64	S32 len = mDelimiter.size();
 65	for (S32 i=0; i<len; i++)
 66	{
 67		if (s[i] != t[i])
 68		{
 69			res = FALSE;
 70			break;
 71		}
 72	}
 73	return res;
 74}
 75
 76LLKeywords::~LLKeywords()
 77{
 78	std::for_each(mWordTokenMap.begin(), mWordTokenMap.end(), DeletePairedPointer());
 79	std::for_each(mLineTokenList.begin(), mLineTokenList.end(), DeletePointer());
 80	std::for_each(mDelimiterTokenList.begin(), mDelimiterTokenList.end(), DeletePointer());
 81}
 82
 83BOOL LLKeywords::loadFromFile( const std::string& filename )
 84{
 85	mLoaded = FALSE;
 86
 87	////////////////////////////////////////////////////////////
 88	// File header
 89
 90	const S32 BUFFER_SIZE = 1024;
 91	char	buffer[BUFFER_SIZE];	/* Flawfinder: ignore */
 92
 93	llifstream file;
 94	file.open(filename);	/* Flawfinder: ignore */
 95	if( file.fail() )
 96	{
 97		llinfos << "LLKeywords::loadFromFile()  Unable to open file: " << filename << llendl;
 98		return mLoaded;
 99	}
100
101	// Identifying string
102	file >> buffer;
103	if( strcmp( buffer, "llkeywords" ) )
104	{
105		llinfos << filename << " does not appear to be a keyword file" << llendl;
106		return mLoaded;
107	}
108
109	// Check file version
110	file >> buffer;
111	U32	version_num;
112	file >> version_num;
113	if( strcmp(buffer, "version") || version_num != (U32)KEYWORD_FILE_CURRENT_VERSION )
114	{
115		llinfos << filename << " does not appear to be a version " << KEYWORD_FILE_CURRENT_VERSION << " keyword file" << llendl;
116		return mLoaded;
117	}
118
119	// start of line (SOL)
120	std::string SOL_COMMENT("#");
121	std::string SOL_WORD("[word ");
122	std::string SOL_LINE("[line ");
123	std::string SOL_ONE_SIDED_DELIMITER("[one_sided_delimiter ");
124	std::string SOL_TWO_SIDED_DELIMITER("[two_sided_delimiter ");
125	std::string SOL_DOUBLE_QUOTATION_MARKS("[double_quotation_marks ");
126
127	LLColor3 cur_color( 1, 0, 0 );
128	LLKeywordToken::TOKEN_TYPE cur_type = LLKeywordToken::WORD;
129
130	while (!file.eof())
131	{
132		buffer[0] = 0;
133		file.getline( buffer, BUFFER_SIZE );
134		std::string line(buffer);
135		if( line.find(SOL_COMMENT) == 0 )
136		{
137			continue;
138		}
139		else if( line.find(SOL_WORD) == 0 )
140		{
141			cur_color = readColor( line.substr(SOL_WORD.size()) );
142			cur_type = LLKeywordToken::WORD;
143			continue;
144		}
145		else if( line.find(SOL_LINE) == 0 )
146		{
147			cur_color = readColor( line.substr(SOL_LINE.size()) );
148			cur_type = LLKeywordToken::LINE;
149			continue;
150		}
151		else if( line.find(SOL_TWO_SIDED_DELIMITER) == 0 )
152		{
153			cur_color = readColor( line.substr(SOL_TWO_SIDED_DELIMITER.size()) );
154			cur_type = LLKeywordToken::TWO_SIDED_DELIMITER;
155			continue;
156		}
157		else if( line.find(SOL_DOUBLE_QUOTATION_MARKS) == 0 )
158		{
159			cur_color = readColor( line.substr(SOL_DOUBLE_QUOTATION_MARKS.size()) );
160			cur_type = LLKeywordToken::DOUBLE_QUOTATION_MARKS;
161			continue;
162		}
163		else if( line.find(SOL_ONE_SIDED_DELIMITER) == 0 )	
164		{
165			cur_color = readColor( line.substr(SOL_ONE_SIDED_DELIMITER.size()) );
166			cur_type = LLKeywordToken::ONE_SIDED_DELIMITER;
167			continue;
168		}
169
170		std::string token_buffer( line );
171		LLStringUtil::trim(token_buffer);
172		
173		typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
174		boost::char_separator<char> sep_word("", " \t");
175		tokenizer word_tokens(token_buffer, sep_word);
176		tokenizer::iterator token_word_iter = word_tokens.begin();
177
178		if( !token_buffer.empty() && token_word_iter != word_tokens.end() )
179		{
180			// first word is the keyword or a left delimiter
181			std::string keyword = (*token_word_iter);
182			LLStringUtil::trim(keyword);
183
184			// second word may be a right delimiter
185			std::string delimiter;
186			if (cur_type == LLKeywordToken::TWO_SIDED_DELIMITER)
187			{
188				while (delimiter.length() == 0 && ++token_word_iter != word_tokens.end())
189				{
190					delimiter = *token_word_iter;
191					LLStringUtil::trim(delimiter);
192				}
193			}
194			else if (cur_type == LLKeywordToken::DOUBLE_QUOTATION_MARKS)
195			{
196				// Closing delimiter is identical to the opening one.
197				delimiter = keyword;
198			}
199
200			// following words are tooltip
201			std::string tool_tip;
202			while (++token_word_iter != word_tokens.end())
203			{
204				tool_tip += (*token_word_iter);
205			}
206			LLStringUtil::trim(tool_tip);
207			
208			if( !tool_tip.empty() )
209			{
210				// Replace : with \n for multi-line tool tips.
211				LLStringUtil::replaceChar( tool_tip, ':', '\n' );
212				addToken(cur_type, keyword, cur_color, tool_tip, delimiter );
213			}
214			else
215			{
216				addToken(cur_type, keyword, cur_color, LLStringUtil::null, delimiter );
217			}
218		}
219	}
220
221	file.close();
222
223	mLoaded = TRUE;
224	return mLoaded;
225}
226
227// Add the token as described
228void LLKeywords::addToken(LLKeywordToken::TOKEN_TYPE type,
229						  const std::string& key_in,
230						  const LLColor3& color,
231						  const std::string& tool_tip_in,
232						  const std::string& delimiter_in)
233{
234	LLWString key = utf8str_to_wstring(key_in);
235	LLWString tool_tip = utf8str_to_wstring(tool_tip_in);
236	LLWString delimiter = utf8str_to_wstring(delimiter_in);
237	switch(type)
238	{
239	case LLKeywordToken::WORD:
240		mWordTokenMap[key] = new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null);
241		break;
242
243	case LLKeywordToken::LINE:
244		mLineTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null));
245		break;
246
247	case LLKeywordToken::TWO_SIDED_DELIMITER:
248	case LLKeywordToken::DOUBLE_QUOTATION_MARKS:
249	case LLKeywordToken::ONE_SIDED_DELIMITER:
250		mDelimiterTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
251		break;
252
253	default:
254		llassert(0);
255	}
256}
257LLKeywords::WStringMapIndex::WStringMapIndex(const WStringMapIndex& other)
258{
259	if(other.mOwner)
260	{
261		copyData(other.mData, other.mLength);
262	}
263	else
264	{
265		mOwner = false;
266		mLength = other.mLength;
267		mData = other.mData;
268	}
269}
270
271LLKeywords::WStringMapIndex::WStringMapIndex(const LLWString& str)
272{
273	copyData(str.data(), str.size());
274}
275
276LLKeywords::WStringMapIndex::WStringMapIndex(const llwchar *start, size_t length):
277mData(start), mLength(length), mOwner(false)
278{
279}
280
281LLKeywords::WStringMapIndex::~WStringMapIndex()
282{
283	if(mOwner)
284		delete[] mData;
285}
286
287void LLKeywords::WStringMapIndex::copyData(const llwchar *start, size_t length)
288{
289	llwchar *data = new llwchar[length];
290	memcpy((void*)data, (const void*)start, length * sizeof(llwchar));
291
292	mOwner = true;
293	mLength = length;
294	mData = data;
295}
296
297bool LLKeywords::WStringMapIndex::operator<(const LLKeywords::WStringMapIndex &other) const
298{
299	// NOTE: Since this is only used to organize a std::map, it doesn't matter if it uses correct collate order or not.
300	// The comparison only needs to strictly order all possible strings, and be stable.
301	
302	bool result = false;
303	const llwchar* self_iter = mData;
304	const llwchar* self_end = mData + mLength;
305	const llwchar* other_iter = other.mData;
306	const llwchar* other_end = other.mData + other.mLength;
307	
308	while(true)
309	{
310		if(other_iter >= other_end)
311		{
312			// We've hit the end of other.
313			// This covers two cases: other being shorter than self, or the strings being equal.
314			// In either case, we want to return false.
315			result = false;
316			break;
317		}
318		else if(self_iter >= self_end)
319		{
320			// self is shorter than other.
321			result = true;
322			break; 
323		}
324		else if(*self_iter != *other_iter)
325		{
326			// The current character differs.  The strings are not equal.
327			result = *self_iter < *other_iter;
328			break;
329		}
330
331		self_iter++;
332		other_iter++;
333	}
334	
335	return result;
336}
337
338LLColor3 LLKeywords::readColor( const std::string& s )
339{
340	F32 r, g, b;
341	r = g = b = 0.0f;
342	S32 values_read = sscanf(s.c_str(), "%f, %f, %f]", &r, &g, &b );
343	if( values_read != 3 )
344	{
345		llinfos << " poorly formed color in keyword file" << llendl;
346	}
347	return LLColor3( r, g, b );
348}
349
350LLFastTimer::DeclareTimer FTM_SYNTAX_COLORING("Syntax Coloring");
351
352// Walk through a string, applying the rules specified by the keyword token list and
353// create a list of color segments.
354void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLWString& wtext, const LLColor4 &defaultColor, LLTextEditor& editor)
355{
356	LLFastTimer ft(FTM_SYNTAX_COLORING);
357	seg_list->clear();
358
359	if( wtext.empty() )
360	{
361		return;
362	}
363	
364	S32 text_len = wtext.size() + 1;
365
366	seg_list->push_back( new LLNormalTextSegment( defaultColor, 0, text_len, editor ) ); 
367
368	const llwchar* base = wtext.c_str();
369	const llwchar* cur = base;
370	const llwchar* line = NULL;
371
372	while( *cur )
373	{
374		if( *cur == '\n' || cur == base )
375		{
376			if( *cur == '\n' )
377			{
378				LLTextSegmentPtr text_segment = new LLLineBreakTextSegment(cur-base);
379				text_segment->setToken( 0 );
380				insertSegment( *seg_list, text_segment, text_len, defaultColor, editor);
381				cur++;
382				if( !*cur || *cur == '\n' )
383				{
384					continue;
385				}
386			}
387
388			// Start of a new line
389			line = cur;
390
391			// Skip white space
392			while( *cur && isspace(*cur) && (*cur != '\n')  )
393			{
394				cur++;
395			}
396			if( !*cur || *cur == '\n' )
397			{
398				continue;
399			}
400
401			// cur is now at the first non-whitespace character of a new line	
402
403			// Line start tokens
404			{
405				BOOL line_done = FALSE;
406				for (token_list_t::iterator iter = mLineTokenList.begin();
407					 iter != mLineTokenList.end(); ++iter)
408				{
409					LLKeywordToken* cur_token = *iter;
410					if( cur_token->isHead( cur ) )
411					{
412						S32 seg_start = cur - base;
413						while( *cur && *cur != '\n' )
414						{
415							// skip the rest of the line
416							cur++;
417						}
418						S32 seg_end = cur - base;
419						
420						//create segments from seg_start to seg_end
421						insertSegments(wtext, *seg_list,cur_token, text_len, seg_start, seg_end, defaultColor, editor);
422						line_done = TRUE; // to break out of second loop.
423						break;
424					}
425				}
426
427				if( line_done )
428				{
429					continue;
430				}
431			}
432		}
433
434		// Skip white space
435		while( *cur && isspace(*cur) && (*cur != '\n')  )
436		{
437			cur++;
438		}
439
440		while( *cur && *cur != '\n' )
441		{
442			// Check against delimiters
443			{
444				S32 seg_start = 0;
445				LLKeywordToken* cur_delimiter = NULL;
446				for (token_list_t::iterator iter = mDelimiterTokenList.begin();
447					 iter != mDelimiterTokenList.end(); ++iter)
448				{
449					LLKeywordToken* delimiter = *iter;
450					if( delimiter->isHead( cur ) )
451					{
452						cur_delimiter = delimiter;
453						break;
454					}
455				}
456
457				if( cur_delimiter )
458				{
459					S32 between_delimiters = 0;
460					S32 seg_end = 0;
461
462					seg_start = cur - base;
463					cur += cur_delimiter->getLengthHead();
464					
465					LLKeywordToken::TOKEN_TYPE type = cur_delimiter->getType();
466					if( type == LLKeywordToken::TWO_SIDED_DELIMITER || type == LLKeywordToken::DOUBLE_QUOTATION_MARKS )
467					{
468						while( *cur && !cur_delimiter->isTail(cur))
469						{
470							// Check for an escape sequence.
471							if (type == LLKeywordToken::DOUBLE_QUOTATION_MARKS && *cur == '\\')
472							{
473								// Count the number of backslashes.
474								S32 num_backslashes = 0;
475								while (*cur == '\\')
476								{
477									num_backslashes++;
478									between_delimiters++;
479									cur++;
480								}
481								// If the next character is the end delimiter?
482								if (cur_delimiter->isTail(cur))
483								{
484									// If there was an odd number of backslashes, then this delimiter
485									// does not end the sequence.
486									if (num_backslashes % 2 == 1)
487									{
488										between_delimiters++;
489										cur++;
490									}
491									else
492									{
493										// This is an end delimiter.
494										break;
495									}
496								}
497							}
498							else
499							{
500								between_delimiters++;
501								cur++;
502							}
503						}
504
505						if( *cur )
506						{
507							cur += cur_delimiter->getLengthHead();
508							seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead() + cur_delimiter->getLengthTail();
509						}
510						else
511						{
512							// eof
513							seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead();
514						}
515					}
516					else
517					{
518						llassert( cur_delimiter->getType() == LLKeywordToken::ONE_SIDED_DELIMITER );
519						// Left side is the delimiter.  Right side is eol or eof.
520						while( *cur && ('\n' != *cur) )
521						{
522							between_delimiters++;
523							cur++;
524						}
525						seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead();
526					}
527
528					insertSegments(wtext, *seg_list,cur_delimiter, text_len, seg_start, seg_end, defaultColor, editor);
529					/*
530					LLTextSegmentPtr text_segment = new LLNormalTextSegment( cur_delimiter->getColor(), seg_start, seg_end, editor );
531					text_segment->setToken( cur_delimiter );
532					insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
533					*/
534					// Note: we don't increment cur, since the end of one delimited seg may be immediately
535					// followed by the start of another one.
536					continue;
537				}
538			}
539
540			// check against words
541			llwchar prev = cur > base ? *(cur-1) : 0;
542			if( !isalnum( prev ) && (prev != '_') )
543			{
544				const llwchar* p = cur;
545				while( isalnum( *p ) || (*p == '_') )
546				{
547					p++;
548				}
549				S32 seg_len = p - cur;
550				if( seg_len > 0 )
551				{
552					WStringMapIndex word( cur, seg_len );
553					word_token_map_t::iterator map_iter = mWordTokenMap.find(word);
554					if( map_iter != mWordTokenMap.end() )
555					{
556						LLKeywordToken* cur_token = map_iter->second;
557						S32 seg_start = cur - base;
558						S32 seg_end = seg_start + seg_len;
559
560						// llinfos << "Seg: [" << word.c_str() << "]" << llendl;
561
562						insertSegments(wtext, *seg_list,cur_token, text_len, seg_start, seg_end, defaultColor, editor);
563					}
564					cur += seg_len; 
565					continue;
566				}
567			}
568
569			if( *cur && *cur != '\n' )
570			{
571				cur++;
572			}
573		}
574	}
575}
576
577void LLKeywords::insertSegments(const LLWString& wtext, std::vector<LLTextSegmentPtr>& seg_list, LLKeywordToken* cur_token, S32 text_len, S32 seg_start, S32 seg_end, const LLColor4 &defaultColor, LLTextEditor& editor )
578{
579	std::string::size_type pos = wtext.find('\n',seg_start);
580	
581	while (pos!=-1 && pos < (std::string::size_type)seg_end)
582	{
583		if (pos!=seg_start)
584		{
585			LLTextSegmentPtr text_segment = new LLNormalTextSegment( cur_token->getColor(), seg_start, pos, editor );
586			text_segment->setToken( cur_token );
587			insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
588		}
589
590		LLTextSegmentPtr text_segment = new LLLineBreakTextSegment(pos);
591		text_segment->setToken( cur_token );
592		insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
593
594		seg_start = pos+1;
595		pos = wtext.find('\n',seg_start);
596	}
597
598	LLTextSegmentPtr text_segment = new LLNormalTextSegment( cur_token->getColor(), seg_start, seg_end, editor );
599	text_segment->setToken( cur_token );
600	insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
601}
602
603void LLKeywords::insertSegment(std::vector<LLTextSegmentPtr>& seg_list, LLTextSegmentPtr new_segment, S32 text_len, const LLColor4 &defaultColor, LLTextEditor& editor )
604{
605	LLTextSegmentPtr last = seg_list.back();
606	S32 new_seg_end = new_segment->getEnd();
607
608	if( new_segment->getStart() == last->getStart() )
609	{
610		seg_list.pop_back();
611	}
612	else
613	{
614		last->setEnd( new_segment->getStart() );
615	}
616	seg_list.push_back( new_segment );
617
618	if( new_seg_end < text_len )
619	{
620		seg_list.push_back( new LLNormalTextSegment( defaultColor, new_seg_end, text_len, editor ) );
621	}
622}
623
624#ifdef _DEBUG
625void LLKeywords::dump()
626{
627	llinfos << "LLKeywords" << llendl;
628
629
630	llinfos << "LLKeywords::sWordTokenMap" << llendl;
631	word_token_map_t::iterator word_token_iter = mWordTokenMap.begin();
632	while( word_token_iter != mWordTokenMap.end() )
633	{
634		LLKeywordToken* word_token = word_token_iter->second;
635		word_token->dump();
636		++word_token_iter;
637	}
638
639	llinfos << "LLKeywords::sLineTokenList" << llendl;
640	for (token_list_t::iterator iter = mLineTokenList.begin();
641		 iter != mLineTokenList.end(); ++iter)
642	{
643		LLKeywordToken* line_token = *iter;
644		line_token->dump();
645	}
646
647
648	llinfos << "LLKeywords::sDelimiterTokenList" << llendl;
649	for (token_list_t::iterator iter = mDelimiterTokenList.begin();
650		 iter != mDelimiterTokenList.end(); ++iter)
651	{
652		LLKeywordToken* delimiter_token = *iter;
653		delimiter_token->dump();
654	}
655}
656
657void LLKeywordToken::dump()
658{
659	llinfos << "[" << 
660		mColor.mV[VX] << ", " <<
661		mColor.mV[VY] << ", " <<
662		mColor.mV[VZ] << "] [" <<
663		wstring_to_utf8str(mToken) << "]" <<
664		llendl;
665}
666
667#endif  // DEBUG